diff --git a/root/wander-train/appraisal_v1.py b/root/wander-train/appraisal_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..2e5f1b2763784ff2840c31cd83226f30449ba456 --- /dev/null +++ b/root/wander-train/appraisal_v1.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""appraisal_v1 - APPRAISAL of a turn from what Oracle7 knows, not from emotion words. + +Scherer's component-process checks, each computed from structure: + EVENT CLASS every noun/verb in the turn is walked up WordNet's hypernym graph; the classes it reaches + (illness, damage, loss, failure, danger, crime/deception, obstruction, gain, praise, reunion, surprise) + are the event's appraisal - "toothache" -> ill_health, "fire" -> danger, "cheated" -> deception, + with no cue for those words anywhere in this file + AGENCY whose doing: the speaker's own ("I broke", "I overslept") vs another's ("you", "he", "they", a passive + "I've been cheated") - another's obstruction appraises to anger, one's own loss to sadness + NOVELTY explicit unexpectedness markers + EMOTION the class those checks produce: fear (danger), anger (another's obstruction / norm violation), + sadness (own loss, illness, damage, failure, regret), surprise (unexpected), happiness (gain, praise, + reunion), disgust (revulsion) +Direct emotion words still count (they are evidence too), but the event walk is what generalises. +usage (dev tuning only): appraisal_v1.py dev""" +import re +import sys +from functools import lru_cache + +try: + from nltk.corpus import wordnet as wn + wn.synsets("dog") +except Exception: # noqa: BLE001 + wn = None + +# WordNet anchor synsets -> appraisal class (a word appraises to a class when one of its senses reaches the anchor) +ANCHORS = { + "danger": ["danger.n.01", "fire.n.04", "emergency.n.01", "accident.n.01", "disaster.n.02", "attack.n.01", + "injury.n.01", "threat.n.01", "violence.n.01"], + "ill_health": ["ill_health.n.01", "illness.n.01", "pain.n.01", "ache.n.01", "symptom.n.01", "infection.n.01", + "wound.n.01"], + "damage": ["damage.n.01", "breakage.n.02", "break.v.02", "break.v.03", "destroy.v.01", "spill.v.01", + "malfunction.n.01"], + "loss": ["loss.n.01", "lose.v.01", "lose.v.02", "death.n.01", "die.v.01", "misplace.v.01", "forget.v.01"], + "failure": ["failure.n.01", "fail.v.02", "fail.v.04", "mistake.n.01", "delay.n.01", "oversleep.v.01"], + "deception": ["deception.n.01", "cheat.v.01", "cheat.v.02", "deceive.v.01", "rip_off.v.01", "crime.n.01", + "rob.v.01", "steal.v.01"], + "obstruction": ["obstruction.n.01", "traffic_jam.n.01", "wait.v.01", "trap.v.01", "block.v.01", + "pollution.n.01", "noise.n.01"], + "gain": ["gain.n.01", "win.v.01", "promotion.n.01", "success.n.01", "prize.n.01", "gift.n.01", "celebration.n.01"], + "praise": ["praise.n.01", "compliment.n.01", "congratulation.n.01"], +} +EMOTION_OF_CLASS = {"danger": "fear", "ill_health": "sadness", "damage": "sadness", "loss": "sadness", + "failure": "sadness", "deception": "anger", "obstruction": "anger", "gain": "happiness", + "praise": "happiness"} +DIRECT = { + "fear": r"\b(?:scared|frighten\w*|terrified|worried|nervous|anxious|panic\w*|butterflies|dread\w*|help me|save (?:my|me)|" + r"(?:i'?m|i am) (?:so |really |very )?afraid of)\b", + "anger": r"\b(?:angry|furious|mad at|annoy\w*|irritat\w*|fed up|sick of|i hate|ridiculous|outrageous|unacceptable|" + r"how dare|damn|can'?t (?:stand|bear)|hundred times|stupid|terrible|horrendous|awful|rude)\b", + "disgust": r"\b(?:disgust\w*|gross|revolting|yuck|nasty|filthy|sickening)\b", + "sadness": r"\b(?:sad|unhappy|depress\w*|lonely|i miss|sorry to hear|so sorry about|heartbroken|upset|cry\w*|" + r"(?:so|very) tired|not up to|against me|(?:i'?m|i am) sorry,? (?:i|sir|madam))\b", + "surprise": r"(?:\bwhat a surprise\b|\bno way\b|\breally\?|\bwow\b|\byikes\b|\bcan'?t believe\b|\bnever believe\b|" + r"\bsurpris\w*|\bfancy meeting\b|\bunbelievable\b|\bwhat happened\b|\balmost didn'?t recogni[sz]e\b|" + r"\bhow come\b|\bis that you\b)", + "happiness": r"\b(?:happy|glad|wonderful|fantastic|excited|delighted|congratulations|thank you so much|can'?t wait|" + r"nice to (?:see|meet)|great news|lovely|beautiful|magnificent|cute|proud|impressed|good job|enjoy|" + r"in heaven|smells good|finally (?:here|back))\b", +} +OTHER_AGENT = re.compile(r"\b(?:you|he|she|they|someone|somebody|that guy|the (?:driver|waiter|company|shop|landlord))\b" + r"|\b(?:been|got|was|were) (?:\w+ed|cheated|robbed|stolen|hit|fired|dumped)\b", re.I) +STOP = set("be have do get go make take come see know say think want like feel look give use find tell ask seem " + "try leave call keep let begin show hear play run move live believe bring happen write sit stand lose pay " + "meet include continue set learn change lead understand watch follow stop create speak read spend grow " + "open walk win offer remember love consider appear buy wait serve die send build stay fall cut reach kill " + "raise pass sell decide return explain hope develop carry break receive agree support hit produce eat " + "cover catch draw choose wear thing time way day man woman people year".split()) + + +@lru_cache(maxsize=None) +def _anchors(): + if wn is None: + return {} + out = {} + for cls, names in ANCHORS.items(): + for n in names: + try: + out[wn.synset(n)] = cls + except Exception: # noqa: BLE001 + pass + return out + + +@lru_cache(maxsize=20000) +def word_class(word): + """the appraisal class reached by the word's MOST FREQUENT sense (WordNet usage counts), walked only when that + sense is a noun - "going" (verb), "down" (adverb), "still" (adverb) are not events; "toothache", "fire" are""" + if wn is None: + return None + anchors = _anchors() + syns = wn.synsets(word.lower()) + if not syns: + return None + top = max(syns, key=lambda x: (sum(l.count() for l in x.lemmas() if l.name().lower() == word.lower()), -syns.index(x))) + if top.pos() != "n": + return None + seen, frontier = set(), [top] + for _ in range(8): + nxt = [] + for x in frontier: + if x in anchors: + return anchors[x] + for h in x.hypernyms() + x.instance_hypernyms(): + if h not in seen: + seen.add(h) + nxt.append(h) + frontier = nxt + if not frontier: + break + return None + + +# events carried by VERBS are named explicitly (a verb's most frequent sense is rarely the eventful one) +EVENT_VERBS = {"deception": r"\b(?:cheated|robbed|stolen|stole|ripped off|scammed|lied to)\b", + "damage": r"\b(?:broke|broken|smashed|spilled|crashed|ruined|wrecked)\b", + "loss": r"\b(?:lost (?:my|our|the|his|her)|left my|passed away|died)\b", + "failure": r"\b(?:overslept|failed|missed (?:the|my)|didn'?t (?:accomplish|finish|make it|pass|win)|forgot)\b", + "obstruction": r"\b(?:stuck|waiting (?:for )?(?:quite|so|such) (?:a )?(?:long|time)|almost hit|traffic jam)\b", + "gain": r"\b(?:won(?!')|promoted|got the job|finally (?:lost|got|finished|passed)|new baby|just bought)\b"} + + +def appraise(text): + """{"emotion", "classes", "agent", "novel", "evidence"} - evidence names the words and the checks that fired""" + t = str(text or "").replace("’", "'").replace(" ' ", "'") + direct = [e for e, rx in DIRECT.items() if re.search(rx, t, re.I)] + classes, ev = [], [] + for c, rx in EVENT_VERBS.items(): + m = re.search(rx, t, re.I) + if m: + classes.append(c) + ev.append("%s->%s" % (m.group(0).lower(), c)) + for w in re.findall(r"[A-Za-z][A-Za-z'-]{2,}", t): + wl = w.lower().strip("'") + if wl in STOP or len(wl) < 4: + continue + c = word_class(wl) + if c: + classes.append(c) + ev.append("%s->%s" % (wl, c)) + other = bool(OTHER_AGENT.search(t)) + novel = "surprise" in direct + emo = "none" + if "fear" in direct or "danger" in classes: + emo = "fear" + elif "anger" in direct or "deception" in classes or ("obstruction" in classes and other): + emo = "anger" + elif "disgust" in direct: + emo = "disgust" + elif "gain" in classes and "finally" in t.lower(): + emo = "happiness" + elif "sadness" in direct or any(c in classes for c in ("ill_health", "loss", "damage", "failure")): + emo = "sadness" + elif novel: + emo = "surprise" + elif "happiness" in direct or any(c in classes for c in ("gain", "praise")): + emo = "happiness" + return {"emotion": emo, "classes": classes, "agent": "other" if other else "self", "novel": novel, + "evidence": ev + ["direct:%s" % d for d in direct]} + + +def _dev(): + import collections + sys.path.insert(0, "/root/w240") + from relational_instrument_v1 import dd + tp, hu, pr, fp, nn, miss = collections.Counter(), collections.Counter(), collections.Counter(), 0, 0, [] + fps = collections.Counter() + for r in dd("dev"): + g, a = r["e0"], appraise(r["opener"]) + p = a["emotion"] + if g != "none": + hu[g] += 1 + else: + nn += 1 + if p != "none": + fp += 1 + fps[(p, tuple(a["evidence"][:3]))] += 1 + if p != "none": + pr[p] += 1 + if g == p and g != "none": + tp[g] += 1 + elif g != "none" and len(miss) < 25: + miss.append((g, p, a["evidence"][:4], r["opener"][:80])) + for e in ("anger", "disgust", "fear", "happiness", "sadness", "surprise"): + print(" %-9s %3d / %3d human / %3d predicted" % (e, tp[e], hu[e], pr[e])) + print(" DEV recall %d/%d | false positives on neutral %d/%d" % (sum(tp.values()), sum(hu.values()), fp, nn)) + print(" top false-positive evidence:", fps.most_common(12)) + for m in miss: + print(" miss", m) + + +if __name__ == "__main__": + _dev() diff --git a/root/wander-train/chat_context_v1.py b/root/wander-train/chat_context_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..ed3c3ae6200fcb971b6d71b829168b7b3aea2efa --- /dev/null +++ b/root/wander-train/chat_context_v1.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""chat_context_v1 - Oracle7's CONVERSATION engine (Ben 2026-10-09: "we have what we need for a proper chat model"). + +Chat failed by REFUSAL, not by error: a follow-up ("Where was he born?", "And its currency?", "And of Denmark?", "What does +that cost?") reached the core without the conversation, and the core either refused or - worse - read the pronoun as the +topic ("he" -> an article about the word "he"). This engine never asserts anything itself. It keeps a small DIALOGUE STATE +per session and REWRITES a turn that only makes sense through the conversation into the standalone question it stands +for; the whole verified chain (router, 210 specialists, fusion, maths) then answers that question exactly as it would if +it had been asked outright. The rewrite is shown in the reply's path and provenance, so the reading can be checked. + + STATE (per session): the previous standalone question; its SUBJECT entity (the entity asked about) and its ANSWER entity + (the entity the reply named), each with whether it is a person; the previous numeric RESULT (value + unit); solved + variables ("x = 5"); the previous question's material phrase ("burning 5 kg of coal"). + PRONOUNS he/she/him/his/her -> the person in play (the answer if it is a person - "Who wrote Frankenstein? / Who was her + mother?" - else the subject); it/its -> the THING asked about ("Which planet does Titan orbit? / Who discovered + it?" is Titan); "that country / that city ..." -> the entity just named; possessives become "the of ". + ELLIPSIS "And its currency?" (relation swap), "And of Denmark?" / "And Doctor Zhivago?" (entity swap in the previous + question), "And on the Moon?" / "And with radius 10 m?" (modifier swap or addition), "Where?" / "In what year?" + (question-word swap, active -> passive when needed: "Who discovered Titan?" -> "In what year was Titan + discovered?"). + RESULTS "What does that cost at $0.20 per kWh?" -> the previous result ("300 kWh"); "What is x squared?" -> the solved + x; "How much CO2 does that emit?" -> the previous material phrase ("burning 5 kg of coal"). + MULTI "What is the capital of Austria and what is its currency?" -> two questions, each resolved and answered. +A turn with nothing to resolve is passed through untouched. If a rewrite cannot be made with confidence, nothing is +rewritten (the chain's own reply stands).""" +import re + +import relation_spec_v1 as RS + +PRON_PERSON = r"he|she|him|his|her|hers" +PRON_THING = r"it|its" +REL_NOUNS = r"capital(?: city)?|currency|population|official language|national anthem|anthem|continent|calling code|highest point|" \ + r"leader|president|prime minister|author|director|composer|creator|founder|headquarters|nationality|religion|" \ + r"spouse|wife|husband|father|mother|birthplace|place of birth|date of birth|birthday" +WH = r"who|what|which|where|when|how" +STATE_CAP = 2000 +UNRESOLVED = "__unresolved__" # rewrite() -> (UNRESOLVED, pronoun): a short turn whose pronoun has nothing to refer to + + +class Turn: + def __init__(self): + self.q = None # previous standalone question + self.subj = None # (qid, label, is_person, surface) + self.ans = None # (qid, label, is_person) + self.result = None # (value, unit) + self.solved = {} # variable -> value + self.material = None # "burning 5 kg of coal" / "5 kg of coal" + + +_STATES = {} + + +def state(sid): + st = _STATES.get(sid) + if st is None: + if len(_STATES) > STATE_CAP: + _STATES.clear() + st = _STATES[sid] = Turn() + return st + + +def _is_person(qid): + try: + return 5 in RS.classes_of(qid)[0] + except Exception: # noqa: BLE001 + return False + + +def _entity_of_question(q): + a = RS.analyse(q) + if not a or not a["cands"]: + return None + qid = a["cands"][0] + return (qid, RS._rs().label(qid) or a["span"], _is_person(qid), a["span"]) + + +RECEIPT_ENDS = re.compile(r"From the Wikidata relation store: (.+?)(?: \(Q(\d+)\))? - [^()]*? \(P\d+\) - (.+?)(?: \(Q(\d+)\))?\.\s*(?:\[|$)") +TYPED = re.compile(r"\(typed relations: wd:Q(\d+) P\d+ wd:Q(\d+)\)") +RECEIPT = re.compile(r"From the Wikidata relation store: .*? - .*? \(P\d+\) - (.+?)(?: \(Q(\d+)\))?\.\s*(?:\[|$)") + + +def _entity_of_reply(reply, prefer_person=False): + """the entity the reply named; after a WHO question the answer is a person, so a person among the name's top candidates + is preferred (the linker's popularity order put a song called "Pablo Picasso" before the painter)""" + text = re.sub(r"^\s*\[ev:[a-z_]+\]\s*", "", str(reply or "")) + m = RECEIPT_ENDS.search(text) + if m: + # the receipt reads subject - property - value, in the direction of the STORE: an inverse read ("Who painted + # Guernica?" -> "Pablo Picasso (Q5593) - wrote - Guernica (Q175036)") puts the answer FIRST. The answer is the end + # whose label the reply leads with. + lead = text.split(". From the Wikidata relation store:", 1)[0].strip().lower() + ends = [(m.group(1), m.group(2)), (m.group(3), m.group(4))] + pick = [e for e in ends if e[1] and e[0].strip().lower() == lead] or [e for e in ends[::-1] if e[1]] + if pick: + qid = int(pick[0][1]) + return (qid, RS._rs().label(qid) or pick[0][0], _is_person(qid)) + m = RECEIPT.search(text) + if m and m.group(2): + qid = int(m.group(2)) + return (qid, RS._rs().label(qid) or m.group(1), _is_person(qid)) + # a sentence ends at . ! ? - but not after an initial ("J. R. R. Tolkien") + first = re.split(r"(? 160: + return None + try: + sp = RS.X.L.link(first, 3) + except Exception: # noqa: BLE001 + return None + if not sp: + return None + span, cs = sp[0] + qid = cs[0][0] + if prefer_person: + people = [c[0] for c in cs[:3] if _is_person(c[0])] + if people: + qid = people[0] + return (qid, RS._rs().label(qid) or span, _is_person(qid)) + + +NUM_UNIT = re.compile(r"(?:is|about|=|of)\s+\$?(-?\d[\d,]*(?:\.\d+)?(?:e[+-]?\d+)?)\s*(kwh|kj|mj|j|kg|g|m|km|s|n|pa|kpa|w|kw|litres?|liters?|l|m³|m3|°c|%)?", re.I) +SOLVED = re.compile(r"\b([a-z])\s*=\s*(-?\d+(?:\.\d+)?)\b") +MATERIAL = re.compile(r"\b((?:burning\s+)?\d[\d,]*(?:\.\d+)?\s*(?:kg|g|tonnes?|litres?|liters?|m3)\s+of\s+[a-z][a-z ]{1,24}?)(?=\s+(?:release|emit|produce|give|make|weigh)|[?.,]|$)", re.I) + + +def _result_of_reply(reply): + text = re.sub(r"^\s*\[ev:[a-z_]+\]\s*", "", str(reply or "")) + m = re.search(r"(?:answer is|is about|takes about|cost(?:s)? (?:is |about )?)\s*\$?(-?\d[\d,]*(?:\.\d+)?(?:e[+-]?\d+)?)\s*([a-zA-Z³%°]+)?", text) + if not m: + m = NUM_UNIT.search(text) + if not m: + return None + try: + v = float(m.group(1).replace(",", "")) + except ValueError: + return None + unit = (m.group(2) or "").lower() + if "$" in text[:max(0, m.start(1)) + 2] and not unit: + unit = "$" + return (v, unit) + + +def update(sid, standalone, reply): + """after the chain answered (or declined) the standalone question: remember what the conversation is about""" + st = state(sid) + numeric = bool(re.search(r"\d", standalone)) and isinstance(reply, dict) and reply.get("status") == "answered" \ + and _result_of_reply(reply.get("response")) is not None + if not recognised(standalone) and not numeric: + return # context comes from FACTUAL turns only + st.q = standalone + # an entity the conversation already resolved KEEPS its identity when the rewrite names it (relinking "Pablo Picasso" + # by popularity picked a song, Q7121722, over the painter the previous answer named, Q5593) + subj = next((tuple(e[:3]) + (e[1],) for e in (st.ans, st.subj) if e and len(e[1] or "") >= 3 and e[1] in standalone), None) \ + or _entity_of_question(standalone) + typed = TYPED.search(str(reply.get("response", ""))) if isinstance(reply, dict) else None + if subj and typed: + for q in (int(typed.group(1)), int(typed.group(2))): + if q != subj[0] and (RS._rs().label(q) or "").lower() == subj[1].lower(): + subj = (q, subj[1], _is_person(q), subj[3]) # the reply's own typed relation names the exact entity + if subj: + st.subj = subj + if isinstance(reply, dict) and reply.get("status") == "answered": + ans = _entity_of_reply(reply.get("response"), prefer_person=bool(re.match(r"^\s*(?:who|whom|whose)\b", standalone, re.I))) + st.ans = ans if ans and (not subj or ans[0] != subj[0]) else None + r = _result_of_reply(reply.get("response")) + st.result = r + for var, val in SOLVED.findall(str(reply.get("response", ""))): + st.solved[var] = val + mm = MATERIAL.search(standalone) + if mm: + st.material = mm.group(1).strip() + + +def _person(st): + if st.ans and st.ans[2]: + return st.ans[1] + if st.subj and st.subj[2]: + return st.subj[1] + return None + + +def _thing(st): + if st.subj and not st.subj[2]: + return st.subj[1] + if st.ans and not st.ans[2]: + return st.ans[1] + return None + + +def _cap(s): + s = s.strip() + return s[:1].upper() + s[1:] if s else s + + +def _q(s): + s = s.strip().rstrip("?.! ") + return _cap(s) + "?" + + +def _swap_wh(prev, wh): + """'When was Alfred Hitchcock born?' + 'Where' -> 'Where was Alfred Hitchcock born?'; active 'Who discovered Titan?' -> + 'In what year was Titan discovered?'""" + p = prev.strip().rstrip("?") + m = re.match(r"^(?:who|what|which|where|when|how)\b(?:\s+year)?\s+((?:was|were|is|are|did|does|do)\b.*)$", p, re.I) + if m: + return _q("%s %s" % (wh, m.group(1))) + m = re.match(r"^who\s+(\w+ed)\s+(.+)$", p, re.I) + if m: + return _q("%s was %s %s" % (wh, m.group(2), m.group(1))) + return None + + +def _rewrite_raw(sid, text): + """-> (kind, standalone question) with kind "num" | "rel", (UNRESOLVED, pronoun), or None""" + st = state(sid) + t = " ".join(str(text or "").split()) + low = t.lower().rstrip("?.! ") + if not t: + return None + pron = re.search(r"\b(he|she|him|his|her|it|its|they|them|their)\b", low) + short_bare = pron is not None and len(low.split()) <= 12 and re.search(r"(?<=\s)[A-Z][a-z]+", t) is None + if not st.q: + return (UNRESOLVED, pron.group(1)) if short_bare and pron.group(1) not in ("it", "its") else None + # results and variables + m = re.search(r"\bwhat is ([a-z]) (squared|cubed)\b", low) + if m and m.group(1) in st.solved: + power = "2" if m.group(2) == "squared" else "3" + return ("num", _q(re.sub(r"\b%s (squared|cubed)\b" % m.group(1), "%s^%s" % (st.solved[m.group(1)], power), t, flags=re.I))) + if re.search(r"\b(?:that|this)\b", low) and (st.result or st.material): + if re.search(r"\b(?:co2|carbon dioxide|emit|emission|release|weigh)", low) and st.material: + return ("num", _q(re.sub(r"\b(?:that|this)\b", st.material, t, count=1, flags=re.I))) + if st.result and st.result[1] and re.search(r"per\s+%s\b" % re.escape(st.result[1]), low): + return ("num", _q(re.sub(r"\b(?:that|this)\b", "%s %s" % (_fmt(st.result[0]), st.result[1]), t, count=1, flags=re.I))) + # bare question word: "Where?", "When?", "In what year?" + m = re.match(r"^(?:and\s+)?(where|when|in what year|what year)$", low) + if m: + r = _swap_wh(st.q, {"what year": "In what year"}.get(m.group(1), _cap(m.group(1)))) + return ("rel", r) if r else None + # ellipsis with "And ..." + m = re.match(r"^(?:and|what about|how about)\s+(.+)$", low) + if m: + rest = t[len(t) - len(m.group(1)) - (len(t) - len(t.rstrip("?.! "))):].rstrip("?.! ") + rest_low = rest.lower() + mr = re.match(r"^(?:the\s+|its\s+|his\s+|her\s+|their\s+)?(%s)$" % REL_NOUNS, rest_low) + if mr: + ent = _thing(st) if not re.match(r"^(his|her)\b", rest_low) else _person(st) + ent = ent or _thing(st) or _person(st) + return ("rel", _q("What is the %s of %s" % (mr.group(1), ent))) if ent else None + mp = re.match(r"^(on|in|at|with|for|of)\s+(.+)$", rest, re.I) + if mp and mp.group(1).lower() in ("of", "for") and st.subj and st.subj[3]: + r = _replace_subject(st, mp.group(2)) + return ("rel", r) if r else None + if mp and mp.group(1).lower() in ("on", "in", "at", "with"): + prep, phrase = mp.group(1).lower(), mp.group(2) + pm = re.search(r"\b%s\s+[^?]*?\d[^?]*?(?=$|\?|,)" % prep, st.q, re.I) if re.search(r"\d", phrase) else None + if not st.result: + return None # a modifier swap continues a COMPUTATION only + if pm: + return ("num", _q(st.q[:pm.start()] + "%s %s" % (prep, phrase) + st.q[pm.end():])) + return ("num", _q(st.q.rstrip("?.! ") + " %s %s" % (prep, phrase))) + if rest[:1].isupper() and st.subj and st.subj[3]: + r = _replace_subject(st, rest) + return ("rel", r) if r else None + return None + # "that country" / possessive and personal pronouns + out = t + changed = False + m = re.search(r"\bthat\s+(country|city|state|planet|nation|island|river|band|team|company|book|film|novel|place)\b", out, re.I) + if m: + ent = (st.ans[1] if st.ans and not st.ans[2] else None) or _thing(st) + if not ent: + return None + out = out[:m.start()] + ent + out[m.end():] + changed = True + # a turn that names its own subject (a capitalised name after the first word) keeps its own pronouns + names_own = re.search(r"(?<=\s)[A-Z][a-z]+", out) is not None and not changed + for pron, getter in (() if names_own else ((PRON_PERSON, _person), (PRON_THING, _thing))): + for pm in list(re.finditer(r"\b(%s)\b(\s+[a-z]+(?:\s+[a-z]+)?)?" % pron, out, re.I))[::-1]: + ent = getter(st) + if not ent: + return (UNRESOLVED, pm.group(1).lower()) if short_bare else None + word = pm.group(1).lower() + if word in ("his", "her", "its") and pm.group(2): + noun = pm.group(2).strip() + if re.match(r"^(%s)\b" % REL_NOUNS, noun, re.I) or word in ("his", "its"): + rep = "the %s of %s" % (noun, ent) + out = out[:pm.start()] + rep + out[pm.end():] + changed = True + continue + out = out[:pm.start(1)] + ent + out[pm.end(1):] + changed = True + if not changed: + return None + return ("rel", _q(out)) + + +def recognised(q): + """a question the verified relation layer can READ: it matches one of the hand-written relation readers whole""" + try: + import relation_spec_v2 as S2 + _, low = S2.normalise(q) + for key in S2.RD.PROPS: + c = S2.compiled(key) + if any(rx.match(low) for d in ("fwd", "inv") for rx in c[d]): + return True + except Exception: # noqa: BLE001 + return False + return False + + +def rewrite(sid, text): + """-> the standalone question this turn stands for, (UNRESOLVED, pronoun), or None. A rewrite is ACCEPTED only when it is + exactly right for the verified layer: a relation rewrite must be a question the relation readers recognise; a numeric + rewrite must continue a numeric answer. Anything else goes to the core untouched - the core has its own conversation + acts ("got it", "why?", "did you run it?", "how does it work?") that a rewrite would break (m52 replay, 2026-10-09).""" + r = _rewrite_raw(sid, text) + if r is None: + return None + if r[0] == UNRESOLVED: + return r if recognised(text) else None + kind, q = r + if not q: + return None + if kind == "num": + return q if state(sid).result or state(sid).solved else None + return q if recognised(q) else None + + +def _replace_subject(st, new): + surface = st.subj[3] + if surface and surface in st.q: + return _q(st.q.replace(surface, new, 1)) + lab = st.subj[1] + if lab and lab in st.q: + return _q(st.q.replace(lab, new, 1)) + return None + + +def _fmt(v): + return ("%d" % v) if abs(v - round(v)) < 1e-9 else ("%.6g" % v) + + +SPLIT = re.compile(r"\s+and\s+(?=(?:%s)\b)" % WH, re.I) + + +def split_parts(text): + """'What is the capital of Austria and what is its currency?' -> two questions (only when the second part is itself a + question starting with a question word)""" + t = " ".join(str(text or "").split()).rstrip("?") + parts = SPLIT.split(t) + if len(parts) < 2: + return None + # split ONLY when every part is a question the relation readers can read (panel v27 2026-10-09: a reading passage + # containing "... and Where to Find Them" and "Everything is fine, and how are you?" were split and lost) + return [_q(p) for p in parts] if all(_shape_ok(_q(p)) for p in parts) else None + + +def _shape_ok(p): + """the part reads as a relation question - as written, or with its pronoun standing for a named thing""" + if recognised(p): + return True + s = re.sub(r"\b(?:its|his|her|their)\s+(\w+(?:\s+\w+)?)", r"the \1 of Peru", p, flags=re.I) + s = re.sub(r"\b(?:he|she|it|they|him|them)\b", "Peru", s, flags=re.I) + return s != p and recognised(s) diff --git a/root/wander-train/chem_agents_v1.py b/root/wander-train/chem_agents_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..bbc6811d00d4c92a86332108bac7a2a830f38e55 --- /dev/null +++ b/root/wander-train/chem_agents_v1.py @@ -0,0 +1,586 @@ +#!/usr/bin/env python3 +"""chem_agents_v1 - five chemistry agents (Oracle7 specialists) that compute exactly what a question asks, from the values it +gives, with every constant stated. Built for the university-chemistry questions the benchmark curriculum showed the family +could not reach (MMLU-Pro chemistry, development rows 3400-3999 only). + GasAgent ideal gas: density, molar mass from density, one unknown of P V n T (mass + formula -> n) + PhAgent pH / pOH / [H+] of strong acids and bases, weak acid or base from Ka / Kb / pKa, Henderson-Hasselbalch + PhotonAgent photon energy / wavelength / frequency (per photon, per mole, eV), de Broglie, hydrogen-like transitions + ColligativeAgent freezing-point depression, boiling-point elevation, molality, molar mass from a shift, osmotic pressure + KineticsAgent first-order decay: half-life, rate constant, time to a fraction, fraction left +LAW (from the specialist network): EVERY number in the question must be read by the computation, or the agent declines - +an unread number means a condition the formula does not know about. Each agent also declines what it cannot type +(unknown formula, missing value, multi-part questions). Returns {kind, status, response, value}.""" +from __future__ import annotations + +import csv +import math +import os +import re + +HERE = os.path.dirname(os.path.abspath(__file__)) +R_LATM = 0.082057366 # L atm / (mol K) +R_J = 8.314462618 # J / (mol K) +H = 6.62607015e-34 # J s (exact) +C = 2.99792458e8 # m / s (exact) +NA = 6.02214076e23 # 1 / mol (exact) +EV = 1.602176634e-19 # J (exact) +ME = 9.1093837015e-31 # kg +RH = 1.09677583e7 # m^-1, Rydberg constant for hydrogen + + +def _table(): + for p in (os.path.join(HERE, "pubchem_periodic_table.csv"), __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/specialists/chemistry-v3/pubchem_periodic_table.csv"): + if os.path.exists(p): + return {r["Symbol"]: float(r["AtomicMass"]) for r in csv.DictReader(open(p, encoding="utf-8"))} + return {} + + +EL = _table() + + +# ---------------------------------------------------------------- text +def clean(t): + t = " ".join(str(t or "").split()) + t = t.replace("−", "-").replace("–", "-").replace("\\times", "×").replace("\\cdot", "·").replace("°", " deg ") + t = re.sub(r"\\(?:mathrm|text|mathbf|operatorname)\s*\{([^{}]*)\}", r"\1", t) + t = re.sub(r"\\(?:mathrm|text)\s*\{([^{}]*)\}", r"\1", t) + t = t.replace("$", " ").replace("~", " ").replace("\\,", " ").replace("{", "").replace("}", "") + t = re.sub(r"\^\s*\\circ|\\circ", " deg ", t) + t = re.sub(r"_(\d)", r"\1", t) # CO_2 -> CO2, H_2SO_4 -> H2SO4 + t = re.sub(r"(? "2×10^-15" + t = re.sub(r"\s+", " ", t) + return t.strip() + + +NUMRE = re.compile(r"(? 0 and re.match(r"[A-Z)\]a-z]", t[a - 1]) and not re.match(r"[a-z]{2,}", t[max(0, a - 3):a]): + continue + if re.match(r"\d", t[a]) and a > 0 and t[a - 1] == "(" and re.match(r"\(\d+\)\s*[A-Z]", t[a - 1:]): + continue + out.append((m.start(), m.end(), _val(m))) + return out + + +class Reader: + """reads typed quantities off the question; records which number spans it used""" + + def __init__(self, text): + self.t = clean(text) + self.low = self.t.lower() + self.nums = numbers(self.t) + self.used = set() + + def _take(self, pattern, flags=re.I): + found = [] + for m in re.finditer(pattern, self.t, flags): + found.append(m) + return found + + def q(self, units, conv): + """all quantities whose unit is one of `units`; conv(value, unit) -> SI-ish value""" + res = [] + pat = r"(" + NUMRE.pattern + r")\s*(" + units + r")(?![a-z])" + for m in re.finditer(pat, self.t, re.I): + nm = NUMRE.match(m.group(1)) + v = _val(nm) + res.append((conv(v, m.group(m.lastindex).lower()), m.start(), m.start() + len(m.group(1)))) + # one value stated twice in two units ("720 mm Hg or 720 torr") is one quantity: keep it once, read both spans + if len(res) > 1 and all(abs(x[0] - res[0][0]) <= 1e-9 * max(abs(res[0][0]), 1e-300) for x in res): + for x in res[1:]: + self.used.add(x[1]) + res = res[:1] + return res + + def use(self, item): + self.used.add(item[1]) + return item[0] + + def all_read(self, also=()): + starts = set(self.used) | set(also) + return all(any(abs(s - a) <= 1 for s in starts) for a, _, _ in self.nums) + + +P_UNITS = r"atm(?:ospheres?)?|torr|mm\s*hg|mmhg|kpa|pa|bar|psi" +V_UNITS = r"liters?|litres?|l|ml|milliliters?|cm3|cm\^3|dm3|dm\^3|m3|m\^3" +T_UNITS = r"deg\s*c|deg\s*k|deg\s*f|k(?![a-z])|kelvin|celsius|c(?![a-z])" +M_UNITS = r"kg|g|mg|grams?|kilograms?|milligrams?" + + +def p_atm(v, u): + u = u.replace(" ", "") + return v * {"atm": 1, "atmosphere": 1, "atmospheres": 1, "torr": 1 / 760, "mmhg": 1 / 760, "kpa": 1 / 101.325, "pa": 1 / 101325, + "bar": 1 / 1.01325, "psi": 1 / 14.6959}[u] + + +def v_l(v, u): + u = u.replace("^", "") + if u in ("ml", "milliliter", "milliliters", "cm3"): + return v / 1000 + if u == "m3": + return v * 1000 + return v + + +def t_k(v, u): + u = u.replace(" ", "") + if u in ("degc", "c", "celsius"): + return v + 273.15 + if u == "degf": + return (v - 32) * 5 / 9 + 273.15 + return v + + +def m_g(v, u): + return v * 1000 if u.startswith("k") else v / 1000 if u.startswith("m") else v + + +def molar_mass(f): + """H2SO4, Ca(OH)2, CuSO4·5H2O -> g/mol; None if any symbol is unknown""" + f = f.replace("·", ".").replace("*", ".") + total = 0.0 + for part in f.split("."): + k = re.match(r"(\d+)(.*)", part) + mult, part = (int(k.group(1)), k.group(2)) if k else (1, part) + stack, cur, i = [], {}, 0 + while i < len(part): + ch = part[i] + if ch == "(": + stack.append(cur) + cur = {} + i += 1 + elif ch == ")": + j = i + 1 + while j < len(part) and part[j].isdigit(): + j += 1 + n = int(part[i + 1:j] or 1) + prev = stack.pop() + for s, c in cur.items(): + prev[s] = prev.get(s, 0) + c * n + cur, i = prev, j + else: + m = re.match(r"([A-Z][a-z]?)(\d*)", part[i:]) + if not m: + return None + s = m.group(1) + if s not in EL: + return None + cur[s] = cur.get(s, 0) + int(m.group(2) or 1) + i += len(m.group(0)) + if stack: + return None + total += mult * sum(EL[s] * c for s, c in cur.items()) + return total + + +FORMULA = re.compile(r"(?= 2) and all(s in EL for s in syms): + out.append((f, m.start(), m.end())) + for name, f in NAMES.items(): + for m in re.finditer(r"\b%s\b" % name, t, re.I): + out.append((f, m.start(), m.end())) + return out + + +def sig(x): + return "%.4g" % x + + +def say(kind, text, value=None, status="answered"): + return {"kind": kind, "status": status, "response": text, "value": value} + + +def multi_part(low): + return bool(re.search(r"\(a\)|\(b\)|\ba\)\s|\bb\)\s|\band (?:also |then )?(?:calculate|find|determine|compute)\b", low)) + + +# ---------------------------------------------------------------- agents +class GasAgent: + def answer(self, text, session_id="default"): + r = Reader(text) + low = r.low + if multi_part(low) or not re.search(r"\bgas\b|\bdensity\b|\bmolar mass\b|\bmolecular weight\b|\bideal\b|\batm\b|\btorr\b|\bmm ?hg\b|\bkpa\b", low): + return None + if re.search(r"van ?der ?waals|real gas|effus|diffus|mean free|root.mean|rms|compressib|adiabat|isotherm|work|heat|" + r"partial pressure|mole fraction|mixture|reaction|react|produced|collected over water|vapor pressure|" + r"balloon|leak|graham|kinetic energy|speed|velocity|collision", low): + return None + P, V, T, M = r.q(P_UNITS, p_atm), r.q(V_UNITS, v_l), r.q(T_UNITS, t_k), r.q(M_UNITS, m_g) + D = r.q(r"g\s*/\s*(?:l|liters?|litres?|dm3)|g\s*l\^?-1|grams? per liter", lambda v, u: v) + N = r.q(r"moles?|mol(?![a-z])", lambda v, u: v) + if any(len(x) > 1 for x in (P, V, T, M, D, N)): + return None + stp = bool(re.search(r"\bstp\b|standard (?:temperature and pressure|conditions)", low)) + fs = formulas(r.t) + ask = None + for k, pat in (("density", r"what is the density|calculate the density|find the density|determine the density|density of .* (?:at|under)"), + ("molar", r"molar mass|molecular weight|molecular mass"), + ("pressure", r"what (?:is the )?pressure|calculate the pressure|find the pressure|what pressure"), + ("volume", r"what (?:is the )?volume|calculate the volume|find the volume|what volume|occupy"), + ("temperature", r"what (?:is the )?temperature|at what temperature|calculate the temperature|find the temperature"), + ("moles", r"how many moles|number of moles|moles of gas"), + ("mass", r"what (?:is the )?mass|how many grams|mass of gas")): + if re.search(pat, low): + ask = k + break + if ask is None: + return None + Tk = r.use(T[0]) if T else (273.15 if stp else None) + Pa = r.use(P[0]) if P else (1.0 if stp else None) + f = fs[0][0] if len({x[0] for x in fs}) == 1 else None + Mm = molar_mass(f) if f else None + if ask == "density": + if D or not (Tk and Pa and Mm): + return None + d = Pa * Mm / (R_LATM * Tk) + out = say("gas_density", "The density of %s is %s g/L (d = PM/RT = %s atm × %s g/mol / (0.082057 L·atm/(mol·K) × %s K))." % (f, sig(d), sig(Pa), sig(Mm), sig(Tk)), d) + elif ask == "molar": + if not (D and Tk and Pa): + return None + d = r.use(D[0]) + mm = d * R_LATM * Tk / Pa + out = say("gas_molar_mass", "The molar mass is %s g/mol (M = dRT/P = %s g/L × 0.082057 × %s K / %s atm)." % (sig(mm), sig(d), sig(Tk), sig(Pa)), mm) + else: + n = r.use(N[0]) if N else None + if n is None and M and Mm: + n = r.use(M[0]) / Mm + Vl = r.use(V[0]) if V else None + known = {"pressure": Pa, "volume": Vl, "temperature": Tk, "moles": n} + if ask == "mass": + if not (Pa and Vl and Tk and Mm) or N or M: + return None + nn = Pa * Vl / (R_LATM * Tk) + out = say("gas_mass", "The mass is %s g (n = PV/RT = %s mol; × %s g/mol for %s)." % (sig(nn * Mm), sig(nn), sig(Mm), f), nn * Mm) + else: + others = {k: v for k, v in known.items() if k != ask} + if any(v is None for v in others.values()): + return None + if ask == "pressure": + v = n * R_LATM * Tk / Vl + out = say("gas_pressure", "The pressure is %s atm (P = nRT/V)." % sig(v), v) + elif ask == "volume": + v = n * R_LATM * Tk / Pa + out = say("gas_volume", "The volume is %s L (V = nRT/P)." % sig(v), v) + elif ask == "temperature": + v = Pa * Vl / (n * R_LATM) + out = say("gas_temperature", "The temperature is %s K (T = PV/nR)." % sig(v), v) + else: + v = Pa * Vl / (R_LATM * Tk) + out = say("gas_moles", "The amount is %s mol (n = PV/RT)." % sig(v), v) + return out if r.all_read() else None + + +STRONG_ACID = {"HCl": 1, "HBr": 1, "HI": 1, "HNO3": 1, "HClO4": 1} +STRONG_BASE = {"NaOH": 1, "KOH": 1, "LiOH": 1, "Ba(OH)2": 2, "Ca(OH)2": 2, "Sr(OH)2": 2} + + +class PhAgent: + def answer(self, text, session_id="default"): + r = Reader(text) + low = r.low + if multi_part(low) or not re.search(r"\bph\b|\bpoh\b|\[h\+\]|\[h3o\+\]|hydrogen ion|hydronium", low): + return None + if re.search(r"titrat|added|adding|mix|mixed|after|equivalence|polyprotic|diprotic|ksp|solubility|hydroly[sz]|salt|" + r"percent|degree of|ionization constant of water|at \d+ deg|temperature", low): + return None + conc = r.q(r"m(?![a-z])|molar|mol/l|moles? per liter|n(?![a-z])", lambda v, u: v) + ka = [(m, m.start()) for m in re.finditer(r"\b(k_?a|k_?b|pk_?a|pk_?b)\s*(?:=|is|of|value of)?\s*(" + NUMRE.pattern + ")", r.t, re.I)] + ask = "pH" if re.search(r"what is the ph|calculate the ph|find the ph|determine the ph|ph of|the ph\b", low) else \ + "pOH" if re.search(r"\bpoh\b", low) and not re.search(r"\bph\b(?! ?=)", low.replace("poh", "")) else \ + "H" if re.search(r"\[h\+\]|\[h3o\+\]|hydrogen ion concentration|hydronium ion concentration", low) else None + if ask is None: + return None + hgiven = re.search(r"(?:hydrogen ion|hydronium(?: ion)?|\[h\+\]|\[h3o\+\])\s*(?:concentration\s*)?(?:is|of|=|equal to)\s*(" + NUMRE.pattern + r")\s*m\b", r.t, re.I) + if ask == "pH" and hgiven and not ka: + h = num(hgiven.group(1)) + r.used.add(hgiven.start(1)) + if not (0 < h <= 10): + return None + return say("ph_from_h", "The pH is %s (pH = -log10[H+] = -log10 %s)." % ("%.2f" % -math.log10(h), sig(h)), -math.log10(h)) if r.all_read() else None + phgiven = re.search(r"\bph (?:of|is|=)\s*(" + NUMRE.pattern + ")", r.t, re.I) + if ask == "H" and phgiven and not conc and not ka: + p = num(phgiven.group(1)) + r.used.add(phgiven.start(1)) + h = 10 ** -p + return say("h_from_ph", "[H+] is %s M (10^-pH with pH %s)." % (sig(h), sig(p)), h) if r.all_read() else None + fs = [x[0] for x in formulas(r.t)] + if len(conc) != 1: + return None + c = r.use(conc[0]) + if not ka: + acid = [f for f in fs if f in STRONG_ACID] + base = [f for f in fs if f in STRONG_BASE] + if len(set(acid + base)) != 1 or c <= 1e-7: + return None + if acid: + h = c * STRONG_ACID[acid[0]] + ph = -math.log10(h) + what = "%s is a strong acid, fully ionized: [H+] = %s M" % (acid[0], sig(h)) + else: + oh = c * STRONG_BASE[base[0]] + ph = 14 + math.log10(oh) + what = "%s is a strong base, fully dissociated: [OH-] = %s M" % (base[0], sig(oh)) + else: + if len(ka) != 1: + return None + m, pos = ka[0] + r.used.add(m.start(2)) + kind, kv = m.group(1).lower().replace("_", ""), num(m.group(2)) + k = 10 ** -kv if kind.startswith("p") else kv + if not (0 < k < 1): + return None + x = (-k + math.sqrt(k * k + 4 * k * c)) / 2 # exact quadratic: x^2 / (c - x) = K + if kind.endswith("b"): + ph = 14 + math.log10(x) + what = "weak base, Kb = %s: [OH-] = %s M from x^2/(C - x) = Kb" % (sig(k), sig(x)) + else: + ph = -math.log10(x) + what = "weak acid, Ka = %s: [H+] = %s M from x^2/(C - x) = Ka" % (sig(k), sig(x)) + h = 10 ** -ph + if not r.all_read(): + return None + if ask == "pH": + return say("ph", "The pH is %s (%s)." % ("%.2f" % ph, what), ph) + if ask == "pOH": + return say("poh", "The pOH is %s (%s)." % ("%.2f" % (14 - ph), what), 14 - ph) + return say("h", "[H+] is %s M (%s)." % (sig(10 ** -ph), what), 10 ** -ph) + + +class PhotonAgent: + LEN = {"nm": 1e-9, "nanometers": 1e-9, "nanometer": 1e-9, "a": 1e-10, "angstroms": 1e-10, "angstrom": 1e-10, "å": 1e-10, + "m": 1, "meters": 1, "cm": 1e-2, "um": 1e-6, "μm": 1e-6, "micrometers": 1e-6, "microns": 1e-6, "pm": 1e-12, "mm": 1e-3} + + def answer(self, text, session_id="default"): + r = Reader(text) + low = r.low + if multi_part(low) or not re.search(r"photon|wavelength|frequency|light|radiation|de ?broglie|transition|emitted|absorbed|quantum", low): + return None + if re.search(r"work function|photoelectric|threshold|stopping|intensity|power|watt|lamp|laser|per second|uncertainty|" + r"box|particle in|harmonic|rotational|vibrational|bond|spin|nmr|esr|epr|zeeman|doppler", low): + return None + if re.search(r"black ?body", low) and re.search(r"(?:maximum|peak|most intense).{0,30}wavelength|wavelength.{0,30}(?:maximum|peak)", low): + T = r.q(T_UNITS, t_k) + if len(T) != 1: + return None + lm = 2.897771955e-3 / r.use(T[0]) + return say("wien", "The peak wavelength is %s m (Wien: λmax = b/T, b = 2.8978e-3 m·K)." % sig(lm), lm) if r.all_read() else None + lam = r.q(r"nm|nanometers?|angstroms?|å|pm|um|μm|micrometers?|microns?|cm(?![-^\d])|mm(?![a-z])|m(?![a-z/^\-])", lambda v, u: v * self.LEN[u]) + nu = r.q(r"hz|s\^?-1|sec\^?-1|/s", lambda v, u: v) + # de Broglie + if re.search(r"de ?broglie|wavelength of (?:an?|the) (?:electron|proton|neutron|particle|ball)", low): + vel = r.q(r"m/s|m s\^?-1|meters? per second", lambda v, u: v) + mass = r.q(M_UNITS, lambda v, u: m_g(v, u) / 1000) + ke = r.q(r"ev|electron ?volts?", lambda v, u: v * EV) + m_ = ME if "electron" in low and not mass else (r.use(mass[0]) if len(mass) == 1 else None) + if m_ is None or len(vel) + len(ke) != 1: + return None + p = m_ * r.use(vel[0]) if vel else math.sqrt(2 * m_ * r.use(ke[0])) + lam_ = H / p + return say("de_broglie", "The de Broglie wavelength is %s m (λ = h/p, p = %s kg·m/s)." % (sig(lam_), sig(p)), lam_) if r.all_read() else None + # hydrogen-like transitions + ns = re.findall(r"n\s*=\s*(\d+)", low) + if len(ns) == 2 and re.search(r"hydrogen|he\+|li2\+|transition|electron (?:falls|drops|jumps|moves)", low): + z = 2 if re.search(r"\bhe\+|helium ion|ionized helium", low) else 3 if re.search(r"li2\+|li\+\+", low) else 1 + if z == 1 and not re.search(r"hydrogen", low): + return None + for m in re.finditer(r"n\s*=\s*(\d+)", r.t, re.I): + r.used.add(m.start(1)) + n1, n2 = sorted(int(x) for x in ns) + if n1 == n2: + return None + wn = RH * z * z * (1 / n1 ** 2 - 1 / n2 ** 2) # m^-1 + E = H * C * wn + if re.search(r"wavelength", low): + v, txt = 1 / wn, "The wavelength is %s m (%s nm), from 1/λ = R_H Z² (1/%d² − 1/%d²)." % (sig(1 / wn), sig(1e9 / wn), n1, n2) + elif re.search(r"frequency", low): + v, txt = wn * C, "The frequency is %s Hz (ν = c R_H Z² (1/%d² − 1/%d²))." % (sig(wn * C), n1, n2) + elif re.search(r"energy", low): + v, txt = E, "The energy is %s J per photon (%s eV), ΔE = hc R_H Z² (1/%d² − 1/%d²)." % (sig(E), sig(E / EV), n1, n2) + elif re.search(r"wave ?number", low): + v, txt = wn / 100, "The wavenumber is %s cm^-1." % sig(wn / 100) + else: + return None + return say("transition", txt, v) if r.all_read() else None + # photon: energy <-> wavelength <-> frequency + if len(lam) + len(nu) != 1: + return None + if lam: + L = r.use(lam[0]) + f = C / L + else: + f = r.use(nu[0]) + L = C / f + E = H * f + if re.search(r"(?:energy|kj|kilojoules?).*(?:per mole|/mol|a mole|one mole|mole of photons)|per mole of photons|einstein", low): + v, txt = E * NA / 1000, "The energy is %s kJ/mol (E = N_A hc/λ)." % sig(E * NA / 1000) + elif re.search(r"\bev\b|electron ?volts?", low) and re.search(r"energy", low): + v, txt = E / EV, "The energy is %s eV (E = hc/λ)." % sig(E / EV) + elif re.search(r"energy", low): + v, txt = E, "The energy is %s J per photon (E = hc/λ = %s J·s × %s m/s / %s m)." % (sig(E), H, C, sig(L)) + elif re.search(r"frequency", low) and lam: + v, txt = f, "The frequency is %s Hz (ν = c/λ)." % sig(f) + elif re.search(r"wavelength", low) and nu: + v, txt = L, "The wavelength is %s m (λ = c/ν)." % sig(L) + else: + return None + return say("photon", txt, v) if r.all_read() else None + + +KF = {"water": 1.86, "benzene": 5.12, "camphor": 37.7, "cyclohexane": 20.0} +KB = {"water": 0.512, "benzene": 2.53} +NONELECTROLYTE = ("glucose", "sucrose", "urea", "sugar", "ethylene glycol", "glycerol", "C6H12O6", "C12H22O11", "CH4N2O", "nonelectrolyte", "non-electrolyte") + + +class ColligativeAgent: + def answer(self, text, session_id="default"): + r = Reader(text) + low = r.low + if multi_part(low) or not re.search(r"freezing|boiling|osmotic|molality|molal", low): + return None + if re.search(r"vapor pressure|raoult|percent|dissociat|association|degree|van.t hoff factor of|dimer|ionized|electrolyte(?! )|" + r"activity|ideal solution", low): + return None + solv = [s for s in KF if s in low] + nonel = any(s.lower() in low for s in NONELECTROLYTE) + if not nonel or len(set(solv)) > 1: + return None + Kf_given = re.search(r"\bk_?f\b[^\d]{0,20}(" + NUMRE.pattern + ")", r.t, re.I) + Kb_given = re.search(r"\bk_?b\b[^\d]{0,20}(" + NUMRE.pattern + ")", r.t, re.I) + masses = r.q(M_UNITS, m_g) + molal = r.q(r"m(?![a-z])|molal", lambda v, u: v) + s = solv[0] if solv else "water" + if "osmotic" in low: + conc = r.q(r"m(?![a-z])|molar|mol/l", lambda v, u: v) + T = r.q(T_UNITS, t_k) + if len(conc) != 1 or len(T) != 1: + return None + pi = r.use(conc[0]) * R_LATM * r.use(T[0]) + return say("osmotic", "The osmotic pressure is %s atm (π = MRT, nonelectrolyte)." % sig(pi), pi) if r.all_read() else None + freezing = "freezing" in low + K = None + if freezing: + if Kf_given: + K = num(Kf_given.group(1)) + r.used.add(Kf_given.start(1)) + else: + K = KF.get(s) + else: + if Kb_given: + K = num(Kb_given.group(1)) + r.used.add(Kb_given.start(1)) + else: + K = KB.get(s) + if K is None: + return None + if len(molal) == 1 and not masses: + m = r.use(molal[0]) + elif len(masses) == 2: + fs = [x[0] for x in formulas(r.t) if x[0] in ("C6H12O6", "C12H22O11", "CH4N2O") or x[0] not in ("H2O",)] + solute = [f for f in fs if f != {"water": "H2O"}.get(s)] + mm = molar_mass(solute[0]) if len(set(solute)) == 1 else None + if mm is None: + return None + a, b = sorted(masses, key=lambda x: x[0]) + m = (r.use(a) / mm) / (r.use(b) / 1000) + else: + return None + dT = K * m + if freezing: + base = {"water": 0.0, "benzene": 5.5, "cyclohexane": 6.5, "camphor": 179.8}[s] + if re.search(r"freezing point (?:of the solution|will be|is)|new freezing|at what temperature.*freez|freezes at", low): + return say("freezing_point", "The freezing point is %s deg C (ΔTf = Kf·m = %s × %s = %s deg C below %s deg C)." % (sig(base - dT), K, sig(m), sig(dT), base), base - dT) if r.all_read() else None + if re.search(r"depression|lowered|by how much|change in", low): + return say("freezing_depression", "The freezing point is lowered by %s deg C (ΔTf = Kf·m = %s × %s)." % (sig(dT), K, sig(m)), dT) if r.all_read() else None + else: + base = {"water": 100.0, "benzene": 80.1}[s] + if re.search(r"boiling point (?:of the solution|will be|is)|new boiling|boils at", low): + return say("boiling_point", "The boiling point is %s deg C (ΔTb = Kb·m = %s × %s = %s deg C above %s deg C)." % (sig(base + dT), K, sig(m), sig(dT), base), base + dT) if r.all_read() else None + if re.search(r"elevation|raised|by how much|change in", low): + return say("boiling_elevation", "The boiling point rises by %s deg C (ΔTb = Kb·m)." % sig(dT), dT) if r.all_read() else None + return None + + +class KineticsAgent: + T = {"s": 1, "sec": 1, "seconds": 1, "second": 1, "min": 60, "minutes": 60, "minute": 60, "h": 3600, "hr": 3600, "hours": 3600, + "hour": 3600, "days": 86400, "day": 86400, "d": 86400, "years": 3.15576e7, "year": 3.15576e7, "yr": 3.15576e7, "y": 3.15576e7} + + def answer(self, text, session_id="default"): + r = Reader(text) + low = r.low + if multi_part(low) or not re.search(r"half-?life|first-?order|decay", low): + return None + if re.search(r"second-?order|zero-?order|activation|arrhenius|temperature|catalyst|equilibrium|counts|dpm|" + r"age of|dating|carbon-14|c-14|14c", low): + return None + tq = r.q(r"seconds?|sec|s(?![a-z])|minutes?|min(?![a-z])|hours?|hrs?|h(?![a-z])|days?|d(?![a-z])|years?|yrs?|y(?![a-z])", + lambda v, u: v * self.T[u]) + pct = r.q(r"%|percent", lambda v, u: v / 100) + frac = re.search(r"\b(one|a) (half|third|fourth|quarter|tenth|eighth|sixteenth)\b", low) + half = re.search(r"half-?life (?:of|is|=)", low) + amt = r.q(r"mci|μci|uci|ci(?![a-z])|bq|mg|kg|g(?![a-z])|grams?", lambda v, u: v) + unit = re.search(r"(" + NUMRE.pattern + r")\s*(mCi|μCi|uCi|Ci|Bq|mg|kg|g|grams?)(?![a-z])", r.t) + if len(tq) == 2 and half and len(amt) == 1 and re.search(r"how much.*(?:remain|left)|(?:remain|left) after", low): + a, b = tq + th, t = (a, b) if a[1] < b[1] else (b, a) + th_v, t_v = r.use(th), r.use(t) + left = r.use(amt[0]) * 0.5 ** (t_v / th_v) + return say("amount_left", "The amount remaining is %s %s (× (1/2)^(t/t½) = (1/2)^%s)." % (sig(left), unit.group(unit.lastindex), sig(t_v / th_v)), left) if r.all_read() else None + if len(tq) == 2 and half and re.search(r"(?:what|how much|what fraction|what percent).*(?:remain|left)", low): + a, b = tq + th, t = (a, b) if a[1] < b[1] else (b, a) + th_v, t_v = r.use(th), r.use(t) + f = 0.5 ** (t_v / th_v) + return say("fraction_left", "The fraction remaining is %s (%s%%), (1/2)^(t/t½)." % (sig(f), sig(100 * f)), f) if r.all_read() else None + if len(tq) == 1 and half and (pct or frac) and re.search(r"how long|how much time|what time", low): + th_v = r.use(tq[0]) + if pct: + p = r.use(pct[0]) + f = p if re.search(r"remain|left", low) else 1 - p + else: + f = {"half": .5, "third": 1 / 3, "fourth": .25, "quarter": .25, "tenth": .1, "eighth": .125, "sixteenth": 1 / 16}[frac.group(2)] + if not re.search(r"remain|left", low): + return None + if not (0 < f < 1): + return None + t = th_v * math.log(f) / math.log(0.5) + return say("time_to_fraction", "It takes %s s (t = t½ · log(f)/log(1/2), f = %s)." % (sig(t), sig(f)), t) if r.all_read() else None + if len(tq) == 1 and half and re.search(r"rate constant|\bk\b", low): + k = math.log(2) / r.use(tq[0]) + return say("rate_constant", "The rate constant is %s s^-1 (k = ln 2 / t½)." % sig(k), k) if r.all_read() else None + return None + + +AGENTS = {"chem-gas": GasAgent, "chem-ph": PhAgent, "chem-photon": PhotonAgent, "chem-colligative": ColligativeAgent, + "chem-kinetics": KineticsAgent} diff --git a/root/wander-train/code_organ_v2.py b/root/wander-train/code_organ_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..8fa0dd7a9d63dfa53b16318d280de196900c8732 --- /dev/null +++ b/root/wander-train/code_organ_v2.py @@ -0,0 +1,474 @@ +"""code_organ_v2 - v1 (MBPP-frozen b2e52a1d) + STRICT mode for a single example: every move named by the +request. Programs as specified free moves over a verified function library, checked by execution; every +verified program joins the library (the stored final forms - continuous learning without retraining). + +REQUEST description + input/output examples (asserts) -> function name, argument values and types, expected outputs, + constants the description names ("at least 4 characters", "the character 'l'") +MOVES typed library functions. The TYPE SYSTEM is the specified structure: a function applies only where its + argument types connect; within that the choice is free. The description's words rank the moves. +SEARCH bottom-up enumeration with OBSERVATIONAL EQUIVALENCE: programs that behave identically on the examples are + kept once, so compositions of 3-4 moves stay tractable. +CHECK the rendered source is executed against every example; nothing is released that did not run and pass. +STORE a verified program becomes a library function (its description words are its keywords, its signature its + type); later requests compose it - measured as the library-growth curve. +Only library templates are ever evaluated; no text from the request is executed.""" +from __future__ import annotations + +import ast +import itertools +import json +import math +import re +from collections import Counter + +# ----------------------------------------------------------------------------------------------------------- types +def tyof(v): + if isinstance(v, bool): + return "bool" + if isinstance(v, int): + return "int" + if isinstance(v, float): + return "float" + if isinstance(v, str): + return "str" + if isinstance(v, list): + return "list" + if isinstance(v, tuple): + return "tuple" + if isinstance(v, dict): + return "dict" + if isinstance(v, set): + return "set" + return "any" + + +GROUP = {"num": {"int", "float"}, "seq": {"list", "tuple", "str"}, "coll": {"list", "tuple", "set"}, + "any": {"bool", "int", "float", "str", "list", "tuple", "dict", "set", "any"}} + + +def fits(t, want): + return t == want or t in GROUP.get(want, ()) + + +# ------------------------------------------------------------------------------------------------------ primitives +class Prim: + __slots__ = ("name", "ins", "out", "fn", "src", "kw") + + def __init__(self, name, ins, out, fn, src, kw): + self.name, self.ins, self.out, self.fn, self.src = name, ins, out, fn, src + self.kw = {w.lower().rstrip("s") for w in kw.split()} # v2: normalised exactly like the request words + + +def _big(x): + if isinstance(x, int) and abs(x) > 10 ** 7: + raise ValueError("too large") + return x + + +def _is_prime(n): + if n < 2: + return False + return all(n % d for d in range(2, int(n ** 0.5) + 1)) + + +def _uniq(xs): + out = [] + for x in xs: + if x not in out: + out.append(x) + return out + + +def _flatten(xs): + out = [] + for x in xs: + out += list(x) if isinstance(x, (list, tuple)) else [x] + return out + + +P_ = [ + # numbers + Prim("add", ("num", "num"), "num", lambda a, b: a + b, "({0} + {1})", "add sum plus total"), + Prim("sub", ("num", "num"), "num", lambda a, b: a - b, "({0} - {1})", "subtract difference minus"), + Prim("mul", ("num", "num"), "num", lambda a, b: _big(a * b), "({0} * {1})", "multiply product times"), + Prim("fdiv", ("num", "num"), "num", lambda a, b: a // b, "({0} // {1})", "divide quotient integer"), + Prim("div", ("num", "num"), "float", lambda a, b: a / b, "({0} / {1})", "divide ratio average"), + Prim("mod", ("num", "num"), "num", lambda a, b: a % b, "({0} % {1})", "remainder modulo divisible"), + Prim("pow", ("num", "num"), "num", lambda a, b: _big(a ** b) if b < 64 else 1 / 0, "({0} ** {1})", "power square cube exponent"), + Prim("sq", ("num",), "num", lambda a: a * a, "({0} ** 2)", "square squares"), + Prim("cube", ("num",), "num", lambda a: a ** 3, "({0} ** 3)", "cube cubes"), + Prim("abs", ("num",), "num", abs, "abs({0})", "absolute"), + Prim("neg", ("num",), "num", lambda a: -a, "(-{0})", "negative negate"), + Prim("half", ("num",), "float", lambda a: a / 2, "({0} / 2)", "half area triangle"), + Prim("sqrt", ("num",), "float", math.sqrt, "math.sqrt({0})", "root square"), + Prim("isqrt", ("int",), "int", math.isqrt, "math.isqrt({0})", "root square integer"), + Prim("fact", ("int",), "int", lambda n: math.factorial(n) if 0 <= n <= 30 else 1 / 0, "math.factorial({0})", "factorial"), + Prim("gcd", ("int", "int"), "int", math.gcd, "math.gcd({0}, {1})", "gcd greatest common divisor"), + Prim("lcm", ("int", "int"), "int", lambda a, b: a * b // math.gcd(a, b) if a and b else 0, "({0} * {1} // math.gcd({0}, {1}))", "lcm least common multiple"), + Prim("isprime", ("int",), "bool", _is_prime, "(({0}) > 1 and all(({0}) % d for d in range(2, int(({0}) ** 0.5) + 1)))", "prime"), + Prim("even", ("int",), "bool", lambda n: n % 2 == 0, "({0} % 2 == 0)", "even"), + Prim("odd", ("int",), "bool", lambda n: n % 2 == 1, "({0} % 2 == 1)", "odd"), + Prim("digits", ("int",), "list", lambda n: [int(c) for c in str(abs(n))], "[int(c) for c in str(abs({0}))]", "digits digit"), + Prim("revnum", ("int",), "int", lambda n: int(str(abs(n))[::-1]), "int(str(abs({0}))[::-1])", "reverse number"), + Prim("bin", ("int",), "str", lambda n: bin(n)[2:], "bin({0})[2:]", "binary"), + Prim("frombin", ("str",), "int", lambda s: int(s, 2), "int({0}, 2)", "binary decimal convert"), + Prim("popcount", ("int",), "int", lambda n: bin(n).count("1"), "bin({0}).count('1')", "set bits count ones"), + Prim("xor", ("int", "int"), "int", lambda a, b: a ^ b, "({0} ^ {1})", "xor bit differ"), + Prim("ispow2", ("int",), "bool", lambda n: n > 0 and n & (n - 1) == 0, "({0} > 0 and {0} & ({0} - 1) == 0)", "power two bit"), + Prim("eq", ("any", "any"), "bool", lambda a, b: a == b, "({0} == {1})", "equal same"), + Prim("lt", ("num", "num"), "bool", lambda a, b: a < b, "({0} < {1})", "less smaller"), + Prim("gt", ("num", "num"), "bool", lambda a, b: a > b, "({0} > {1})", "greater larger"), + Prim("not", ("bool",), "bool", lambda a: not a, "(not {0})", "not"), + Prim("and", ("bool", "bool"), "bool", lambda a, b: a and b, "({0} and {1})", "and both"), + Prim("or", ("bool", "bool"), "bool", lambda a, b: a or b, "({0} or {1})", "or either"), + Prim("maxn", ("num", "num"), "num", max, "max({0}, {1})", "maximum largest bigger"), + Prim("minn", ("num", "num"), "num", min, "min({0}, {1})", "minimum smallest"), + Prim("round2", ("num",), "float", lambda a: round(a, 2), "round({0}, 2)", "round"), + Prim("int", ("any",), "int", int, "int({0})", "integer convert"), + Prim("str", ("any",), "str", str, "str({0})", "string convert"), + Prim("float", ("num",), "float", float, "float({0})", "float"), + Prim("tri", ("int",), "int", lambda n: n * (n + 1) // 2, "({0} * ({0} + 1) // 2)", "triangular sum natural"), + Prim("pi_r2", ("num",), "float", lambda r: math.pi * r * r, "(math.pi * {0} * {0})", "area circle"), + Prim("circ", ("num",), "float", lambda r: 2 * math.pi * r, "(2 * math.pi * {0})", "circumference perimeter circle"), + # sequences + Prim("len", ("seq",), "int", len, "len({0})", "length count number size"), + Prim("sum", ("coll",), "num", lambda xs: sum(xs), "sum({0})", "sum total add"), + Prim("max", ("coll",), "any", max, "max({0})", "maximum largest biggest max longest"), + Prim("min", ("coll",), "any", min, "min({0})", "minimum smallest min"), + Prim("sorted", ("coll",), "list", sorted, "sorted({0})", "sort sorted ascending order"), + Prim("sorted_desc", ("coll",), "list", lambda xs: sorted(xs, reverse=True), "sorted({0}, reverse=True)", "sort descending largest"), + Prim("rev", ("seq",), "seq", lambda xs: xs[::-1], "{0}[::-1]", "reverse reversed"), + Prim("first", ("seq",), "any", lambda xs: xs[0], "{0}[0]", "first"), + Prim("last", ("seq",), "any", lambda xs: xs[-1], "{0}[-1]", "last"), + Prim("take", ("seq", "int"), "seq", lambda xs, n: xs[:n], "{0}[:{1}]", "first n smallest largest take"), + Prim("drop", ("seq", "int"), "seq", lambda xs, n: xs[n:], "{0}[{1}:]", "remove first skip"), + Prim("at", ("seq", "int"), "any", lambda xs, i: xs[i], "{0}[{1}]", "nth index element second third"), + Prim("tolist", ("any",), "list", list, "list({0})", "list convert"), + Prim("totuple", ("any",), "tuple", tuple, "tuple({0})", "tuple convert"), + Prim("toset", ("coll",), "set", set, "set({0})", "unique distinct set"), + Prim("uniq", ("coll",), "list", _uniq, "list(dict.fromkeys({0}))", "unique distinct duplicates remove"), + Prim("inter", ("coll", "coll"), "tuple", lambda a, b: tuple(sorted(set(a) & set(b))), "tuple(sorted(set({0}) & set({1})))", "common shared similar intersection"), + Prim("union", ("coll", "coll"), "tuple", lambda a, b: tuple(sorted(set(a) | set(b))), "tuple(sorted(set({0}) | set({1})))", "union combine"), + Prim("diff", ("coll", "coll"), "list", lambda a, b: [x for x in a if x not in b], "[x for x in {0} if x not in {1}]", "difference not in"), + Prim("concat", ("seq", "seq"), "seq", lambda a, b: a + b, "({0} + {1})", "concatenate join append combine merge"), + Prim("count", ("seq", "any"), "int", lambda xs, x: xs.count(x), "{0}.count({1})", "count occurrences frequency"), + Prim("index", ("seq", "any"), "int", lambda xs, x: xs.index(x), "{0}.index({1})", "index position find"), + Prim("contains", ("any", "seq"), "bool", lambda x, xs: x in xs, "({0} in {1})", "contains present exists check"), + Prim("allsame", ("coll",), "bool", lambda xs: len(set(xs)) <= 1, "(len(set({0})) <= 1)", "same equal all identical"), + Prim("alluniq", ("coll",), "bool", lambda xs: len(set(xs)) == len(xs), "(len(set({0})) == len(list({0})))", "unique distinct duplicate"), + Prim("issorted", ("coll",), "bool", lambda xs: list(xs) == sorted(xs), "(list({0}) == sorted({0}))", "sorted order check"), + Prim("prod", ("coll",), "num", lambda xs: _big(math.prod(xs)), "math.prod({0})", "product multiply"), + Prim("mean", ("coll",), "float", lambda xs: sum(xs) / len(xs), "(sum({0}) / len({0}))", "average mean"), + Prim("map_sq", ("coll",), "list", lambda xs: [x * x for x in xs], "[x * x for x in {0}]", "square squares each"), + Prim("map_cube", ("coll",), "list", lambda xs: [x ** 3 for x in xs], "[x ** 3 for x in {0}]", "cube cubes each"), + Prim("map_neg", ("coll",), "list", lambda xs: [-x for x in xs], "[-x for x in {0}]", "negative"), + Prim("map_abs", ("coll",), "list", lambda xs: [abs(x) for x in xs], "[abs(x) for x in {0}]", "absolute"), + Prim("map_str", ("coll",), "list", lambda xs: [str(x) for x in xs], "[str(x) for x in {0}]", "string convert"), + Prim("map_int", ("coll",), "list", lambda xs: [int(x) for x in xs], "[int(x) for x in {0}]", "integer convert"), + Prim("map_len", ("coll",), "list", lambda xs: [len(x) for x in xs], "[len(x) for x in {0}]", "length each"), + Prim("map_first", ("coll",), "list", lambda xs: [x[0] for x in xs], "[x[0] for x in {0}]", "first each"), + Prim("map_last", ("coll",), "list", lambda xs: [x[-1] for x in xs], "[x[-1] for x in {0}]", "last each"), + Prim("map_sum", ("coll",), "list", lambda xs: [sum(x) for x in xs], "[sum(x) for x in {0}]", "sum each"), + Prim("map_add", ("coll", "num"), "list", lambda xs, k: [x + k for x in xs], "[x + {1} for x in {0}]", "add each"), + Prim("map_mul", ("coll", "num"), "list", lambda xs, k: [x * k for x in xs], "[x * {1} for x in {0}]", "multiply each double doubled twice"), + Prim("zipadd", ("coll", "coll"), "list", lambda a, b: [x + y for x, y in zip(a, b)], "[x + y for x, y in zip({0}, {1})]", "add two lists elementwise"), + Prim("zipmul", ("coll", "coll"), "list", lambda a, b: [x * y for x, y in zip(a, b)], "[x * y for x, y in zip({0}, {1})]", "multiply two lists"), + Prim("zipsub", ("coll", "coll"), "list", lambda a, b: [x - y for x, y in zip(a, b)], "[x - y for x, y in zip({0}, {1})]", "subtract two lists"), + Prim("zipdiv", ("coll", "coll"), "list", lambda a, b: [x / y for x, y in zip(a, b)], "[x / y for x, y in zip({0}, {1})]", "divide two lists"), + Prim("zip", ("coll", "coll"), "list", lambda a, b: list(zip(a, b)), "list(zip({0}, {1}))", "pair zip"), + Prim("f_even", ("coll",), "list", lambda xs: [x for x in xs if x % 2 == 0], "[x for x in {0} if x % 2 == 0]", "even"), + Prim("f_odd", ("coll",), "list", lambda xs: [x for x in xs if x % 2 == 1], "[x for x in {0} if x % 2 == 1]", "odd"), + Prim("f_pos", ("coll",), "list", lambda xs: [x for x in xs if x > 0], "[x for x in {0} if x > 0]", "positive"), + Prim("f_neg", ("coll",), "list", lambda xs: [x for x in xs if x < 0], "[x for x in {0} if x < 0]", "negative"), + Prim("f_gt", ("coll", "num"), "list", lambda xs, k: [x for x in xs if x > k], "[x for x in {0} if x > {1}]", "greater than"), + Prim("f_lenge", ("coll", "int"), "list", lambda xs, k: [x for x in xs if len(x) >= k], "[x for x in {0} if len(x) >= {1}]", "at least long characters"), + Prim("f_neq", ("coll", "any"), "list", lambda xs, k: [x for x in xs if x != k], "[x for x in {0} if x != {1}]", "remove all"), + Prim("cumsum", ("coll",), "list", lambda xs: list(itertools.accumulate(xs)), "list(itertools.accumulate({0}))", "cumulative running sum"), + Prim("flatten", ("coll",), "list", _flatten, "[y for x in {0} for y in (x if isinstance(x, (list, tuple)) else [x])]", "flatten nested"), + Prim("mostcommon", ("coll",), "any", lambda xs: Counter(xs).most_common(1)[0][0], "collections.Counter({0}).most_common(1)[0][0]", "most common frequent maximum occurrences"), + Prim("freq", ("coll",), "dict", lambda xs: dict(Counter(xs)), "dict(collections.Counter({0}))", "frequency count each"), + Prim("sortkey1", ("coll",), "list", lambda xs: sorted(xs, key=lambda t: t[1]), "sorted({0}, key=lambda t: t[1])", "sort second element"), + Prim("maxkey_len", ("coll",), "any", lambda xs: max(xs, key=len), "max({0}, key=len)", "longest maximum length"), + Prim("minkey_len", ("coll",), "any", lambda xs: min(xs, key=len), "min({0}, key=len)", "shortest minimum length"), + Prim("range1", ("int",), "list", lambda n: list(range(1, n + 1)) if n <= 10 ** 5 else 1 / 0, "list(range(1, {0} + 1))", "first n natural numbers range"), + Prim("range0", ("int",), "list", lambda n: list(range(n)) if n <= 10 ** 5 else 1 / 0, "list(range({0}))", "range"), + # strings + Prim("upper", ("str",), "str", str.upper, "{0}.upper()", "upper uppercase capital"), + Prim("lower", ("str",), "str", str.lower, "{0}.lower()", "lower lowercase"), + Prim("title", ("str",), "str", str.title, "{0}.title()", "title capitalize first letter each word"), + Prim("capitalize", ("str",), "str", str.capitalize, "{0}.capitalize()", "capitalize"), + Prim("swapcase", ("str",), "str", str.swapcase, "{0}.swapcase()", "toggle swap case"), + Prim("strip", ("str",), "str", str.strip, "{0}.strip()", "strip trim whitespace spaces"), + Prim("nospace", ("str",), "str", lambda s: s.replace(" ", ""), "{0}.replace(' ', '')", "remove spaces whitespace"), + Prim("words", ("str",), "list", str.split, "{0}.split()", "words split"), + Prim("splitby", ("str", "str"), "list", lambda s, d: s.split(d), "{0}.split({1})", "split delimiter"), + Prim("joinsp", ("coll",), "str", lambda xs: " ".join(xs), "' '.join({0})", "join words"), + Prim("join", ("coll",), "str", lambda xs: "".join(xs), "''.join({0})", "join concatenate characters"), + Prim("replace", ("str", "str", "str"), "str", lambda s, a, b: s.replace(a, b), "{0}.replace({1}, {2})", "replace"), + Prim("remove", ("str", "str"), "str", lambda s, a: s.replace(a, ""), "{0}.replace({1}, '')", "remove delete"), + Prim("remove1", ("str", "str"), "str", lambda s, a: s.replace(a, "", 1), "{0}.replace({1}, '', 1)", "remove first occurrence"), + Prim("removelast", ("str", "str"), "str", lambda s, a: (s[:s.rfind(a)] + s[s.rfind(a) + 1:]) if a in s else s, "(({0}[:{0}.rfind({1})] + {0}[{0}.rfind({1}) + 1:]) if {1} in {0} else {0})", "remove last occurrence"), + Prim("chars", ("str",), "list", list, "list({0})", "characters chars"), + Prim("isdigit", ("str",), "bool", str.isdigit, "{0}.isdigit()", "digit numeric integer"), + Prim("isalpha", ("str",), "bool", str.isalpha, "{0}.isalpha()", "alphabet letters"), + Prim("isupper", ("str",), "bool", str.isupper, "{0}.isupper()", "upper"), + Prim("islower", ("str",), "bool", str.islower, "{0}.islower()", "lower"), + Prim("startswith", ("str", "str"), "bool", str.startswith, "{0}.startswith({1})", "starts begins"), + Prim("endswith", ("str", "str"), "bool", str.endswith, "{0}.endswith({1})", "ends"), + Prim("find", ("str", "str"), "int", str.find, "{0}.find({1})", "find position index occurrence"), + Prim("f_upper", ("str",), "str", lambda s: "".join(c for c in s if c.isupper()), "''.join(c for c in {0} if c.isupper())", "uppercase characters extract"), + Prim("f_lower", ("str",), "str", lambda s: "".join(c for c in s if c.islower()), "''.join(c for c in {0} if c.islower())", "lowercase characters remove"), + Prim("f_digit", ("str",), "str", lambda s: "".join(c for c in s if c.isdigit()), "''.join(c for c in {0} if c.isdigit())", "digits numbers extract"), + Prim("f_alpha", ("str",), "str", lambda s: "".join(c for c in s if c.isalpha()), "''.join(c for c in {0} if c.isalpha())", "letters remove non alphabet"), + Prim("f_alnum", ("str",), "str", lambda s: "".join(c for c in s if c.isalnum()), "''.join(c for c in {0} if c.isalnum())", "alphanumeric remove special characters"), + Prim("vowels", ("str",), "int", lambda s: sum(c in "aeiouAEIOU" for c in s), "sum(c in 'aeiouAEIOU' for c in {0})", "vowels count"), + Prim("ispal", ("seq",), "bool", lambda s: s == s[::-1], "({0} == {0}[::-1])", "palindrome"), + Prim("sortchars", ("str",), "str", lambda s: "".join(sorted(s)), "''.join(sorted({0}))", "sort characters"), + Prim("rev_words", ("str",), "str", lambda s: " ".join(s.split()[::-1]), "' '.join({0}.split()[::-1])", "reverse words"), + Prim("ord", ("str",), "int", lambda s: ord(s) if len(s) == 1 else 1 / 0, "ord({0})", "ascii value character"), + Prim("chr", ("int",), "str", lambda n: chr(n) if 0 <= n < 1114111 else 1 / 0, "chr({0})", "character ascii"), + Prim("strlen_words", ("str",), "list", lambda s: [len(w) for w in s.split()], "[len(w) for w in {0}.split()]", "length words"), + Prim("firstchar", ("str",), "str", lambda s: s[0], "{0}[0]", "first character"), + Prim("lastchar", ("str",), "str", lambda s: s[-1], "{0}[-1]", "last character"), + Prim("upper_first_last", ("str",), "str", lambda s: " ".join((w[0].upper() + w[1:-1] + w[-1].upper()) if len(w) > 1 else w.upper() for w in s.split()), "' '.join((w[0].upper() + w[1:-1] + w[-1].upper()) if len(w) > 1 else w.upper() for w in {0}.split())", "capitalize first last letters each word"), +] +# v2 library entries added from the gate panel's FAILURE CLASSES (general-purpose operations; checked on MBPP dev) +P_ += [ + Prim("c2f", ("num",), "float", lambda c: c * 9 / 5 + 32, "({0} * 9 / 5 + 32)", "celsius fahrenheit convert temperature"), + Prim("f2c", ("num",), "float", lambda f: (f - 32) * 5 / 9, "(({0} - 32) * 5 / 9)", "fahrenheit celsius convert temperature"), + Prim("isleap", ("int",), "bool", lambda y: y % 4 == 0 and (y % 100 != 0 or y % 400 == 0), + "({0} % 4 == 0 and ({0} % 100 != 0 or {0} % 400 == 0))", "leap year"), + Prim("joincomma", ("coll",), "str", lambda xs: ",".join(xs), "','.join({0})", "join comma separated"), + Prim("nwords", ("str",), "int", lambda s: len(s.split()), "len({0}.split())", "count words number"), +] +IMPORTS = "import math\nimport itertools\nimport collections\n" + + +# --------------------------------------------------------------------------------------------------------- request +def parse_tests(tests, setup=""): + """'assert f(a, b) == v' -> (fname, [(args, expected)], wrapper-src or None)""" + fname, cases, wrap = None, [], None + for t in tests: + try: + node = ast.parse(t.strip()).body[0] + except SyntaxError: + return None + if not isinstance(node, ast.Assert): + return None + test = node.test + if isinstance(test, ast.Call) or (isinstance(test, ast.UnaryOp) and isinstance(test.operand, ast.Call)): + neg = isinstance(test, ast.UnaryOp) + call = test.operand if neg else test + exp = not neg + lhs = call + elif isinstance(test, ast.Compare) and len(test.ops) == 1 and isinstance(test.ops[0], ast.Eq): + lhs = test.left + try: + exp = ast.literal_eval(test.comparators[0]) + except Exception: # noqa: BLE001 + return None + else: + return None + call, w = lhs, None + if isinstance(lhs, ast.Call) and isinstance(lhs.func, ast.Name) and lhs.func.id in ("set", "sorted", "list", "tuple", "round", "abs") and lhs.args and isinstance(lhs.args[0], ast.Call): + w, call = lhs.func.id, lhs.args[0] + if not (isinstance(call, ast.Call) and isinstance(call.func, ast.Name)): + return None + try: + args = [ast.literal_eval(a) for a in call.args] + except Exception: # noqa: BLE001 + return None + if fname and call.func.id != fname: + return None + fname, wrap = call.func.id, w or wrap + cases.append((args, exp)) + return (fname, cases, wrap) if fname and cases else None + + +WRAP = {None: lambda v: v, "set": set, "sorted": sorted, "list": list, "tuple": tuple, "round": round, "abs": abs} + + +ORDINAL = {"first": 0, "second": 1, "third": 2, "fourth": 3, "last": -1, "double": 2, "doubled": 2, "twice": 2, + "triple": 3, "tripled": 3, "half": 2, "halve": 2, "halved": 2} + + +def consts_of(desc, cases, strict=False): + """constants a program may use. STRICT (one example): only what the request itself names - its numbers, its + quoted characters and its ordinals (first=0, second=1, last=-1) - so no positional coincidence is reachable""" + text = re.split(r"\bso\b", desc or "")[0] if strict else (desc or "") + named = [int(n) for n in re.findall(r"\b(\d{1,4})\b", text)] + named += re.findall(r"'([^']{1,3})'", text) + named += [v for w, v in ORDINAL.items() if re.search(r"\b%s\b" % w, text, re.I)] + out = named if strict else [0, 1, 2, -1, "", " "] + named + return list(dict.fromkeys(out)) + + +GENERIC = {"write", "function", "python", "find", "given", "number", "numbers", "integer", "list", "string", "element", + "elements", "value", "values", "check", "whether", "two", "the", "and", "from", "all", "using", "which", "that", + "with", "into", "for", "each", "array", "tuple", "item", "items", "get", "given", "not", "out", "sequence"} + + +def words_of(desc): + return {w.lower().rstrip("s") for w in re.findall(r"[A-Za-z]+", desc or "") + if len(w) >= 3 and w.lower() not in GENERIC and w.lower().rstrip("s") not in GENERIC} + + +# ----------------------------------------------------------------------------------------------------------- search +class CodeOrgan: + def __init__(self, library=None, max_size=4, cap=60000): + self.prims = list(P_) + list(library or []) + self.max_size, self.cap = max_size, cap + self.why = "" + + def learn(self, name, prog, arity_types, out_t, desc): + """STORE: a verified program becomes a library function; later requests compose it""" + fnc = prog["fn"] + src = "%s(%s)" % (name, ", ".join("{%d}" % i for i in range(len(arity_types)))) + p = Prim("lib_" + name, tuple(arity_types), out_t, lambda *vals: fnc(vals), src, " ".join(words_of(desc))) + self.prims.append(p) + self.library_src = getattr(self, "library_src", {}) + self.library_src[name] = prog["def"].replace(IMPORTS, "") + return p + + def solve(self, desc, tests, setup="", visible=None, strict=False): + self.why = "" + pt = parse_tests(tests, setup) + if not pt: + self.why = "the examples are not simple asserts" + return None + fname, cases, wrap = pt + vis = cases if visible is None else cases[:visible] + if not vis: + return None + W = WRAP[wrap] + nargs = len(vis[0][0]) + if any(len(a) != nargs for a, _ in cases): + self.why = "varying arity" + return None + target = [W(e) if wrap else e for _, e in vis] + kw = words_of(desc) + # the request SPECIFIES the moves: rank by the description's words; keep the whole library but try relevant first + # a stored program is a move only with >= 2 examples, and never inside its own definition; with ONE example + # (strict) only the base library composes - forms verified on one example are too weak to build on from one more + pool = [p for p in self.prims if p.name != "lib_" + fname and not (strict and p.name.startswith("lib_"))] + prims = sorted(pool, key=lambda p: -len(p.kw & kw)) + # terminals: arguments and constants; a term = (src, type, outputs-tuple, fn) + terms_by_size = {1: []} + seen = set() + + def add(size, src, outs, fnc, used=frozenset()): + key = repr(outs) + try: + match = [W(o) if wrap else o for o in outs] == target + except Exception: # noqa: BLE001 + match = False + if match: + # the request SPECIFIES the moves: a fit made only of moves the description never names is a + # coincidence of three examples, not an answer - it is not accepted and does not claim the signature. + # A fit must also READ its inputs: every argument is used, except an 'n' that only restates a length. + reads = all(argn[i] in re.findall(r"a\d+", src) or restates_len[i] for i in range(nargs)) + # STRICT (one example): EVERY operation must be named by the request - the description specifies the + # program and the single example only checks it + named = all(pr.kw & kw for pr in used) if strict else any(pr.kw & kw for pr in used) + if used and reads and named: + return (src, fnc) + self.coincidences = getattr(self, "coincidences", 0) + 1 + return None + if key in seen or len(key) > 1500 or len(seen) > self.cap: + return None # bounded memory: huge values and past-budget terms are not kept + seen.add(key) + ty = tyof(outs[0]) + if any(tyof(o) != ty for o in outs): + ty = "any" + terms_by_size.setdefault(size, []).append((src, ty, outs, fnc, used)) + return None + + argn = ["a%d" % i for i in range(nargs)] + restates_len = [all(isinstance(a[i], int) and any(isinstance(a[j], (list, tuple, str)) and len(a[j]) == a[i] + for j in range(nargs) if j != i) for a, _ in vis) + for i in range(nargs)] + for i in range(nargs): + hit = add(1, argn[i], tuple(a[i] for a, _ in vis), (lambda i: lambda xs: xs[i])(i)) + if hit: + return self._emit(fname, argn, hit, cases, wrap, desc) + for c in consts_of(desc, vis, strict): + add(1, repr(c), tuple(c for _ in vis), (lambda c: lambda xs: c)(c)) + n_terms = 0 + best = None + for size in range(2, self.max_size + 1): + for p in prims: + k = len(p.ins) + # child sizes summing to size-1 + for parts in _compositions(size - 1, k): + pools = [terms_by_size.get(s, []) for s in parts] + pools = [[t for t in pool if fits(t[1], want) or t[1] == "any"] for pool, want in zip(pools, p.ins)] + if any(not pool for pool in pools): + continue + for combo in itertools.product(*pools): + n_terms += 1 + if n_terms > self.cap: + self.why = "search budget exhausted" + return None + try: + outs = tuple(p.fn(*[t[2][j] for t in combo]) for j in range(len(vis))) + except Exception: # noqa: BLE001 + continue + if any(_oversize(o) for o in outs): + continue # bounded memory: never represent a huge intermediate value + src = p.src.format(*[t[0] for t in combo]) + fnc = (lambda p, combo: lambda xs: p.fn(*[t[3](xs) for t in combo]))(p, combo) + used = frozenset([p]).union(*[t[4] for t in combo]) + hit = add(size, src, outs, fnc, used) + if hit: + if not strict: + return self._emit(fname, argn, hit, cases, wrap, desc) + # COVERAGE: how much of what the request says the program's moves account for, jointly + named = len(set().union(*[pr.kw for pr in used]) & kw) + if best is None or named > best[0]: + best = (named, hit) + if strict and best: # v2 STRICT: the fit covering most of the request, any size + return self._emit(fname, argn, best[1], cases, wrap, desc) + self.why = "no program of size <= %d fits the examples" % self.max_size + return None + + def _emit(self, fname, argn, hit, cases, wrap, desc): + src, fnc = hit + lib = getattr(self, "library_src", {}) + used = [n for n in lib if re.search(r"\b%s\(" % re.escape(n), src)] + deps = "".join(lib[n] for n in used) # stored forms this program composes + body = "%s%sdef %s(%s):\n return %s\n" % (IMPORTS, deps, fname, ", ".join(argn), src) + # CHECK: execute the RENDERED source (not the in-memory closure) against every example given + ns = {} + exec(compile(body, "", "exec"), ns) # noqa: S102 - composed only of library templates + W = WRAP[wrap] + passed = 0 + for a, e in cases: + try: + out = ns[fname](*a) + ok = (W(out) if wrap else out) == e + except Exception: # noqa: BLE001 + ok = False + passed += ok + return {"def": body, "fname": fname, "expr": src, "passed": passed, "cases": len(cases), "fn": fnc, + "arity": len(argn), "reused": used} + + +def _oversize(o): + if isinstance(o, bool): + return False + if isinstance(o, int): + return abs(o) > 10 ** 12 + if isinstance(o, str): + return len(o) > 400 + if isinstance(o, (list, tuple, set, dict)): + return len(o) > 200 or any(_oversize(x) for x in list(o)[:50]) + return False + + +def _compositions(total, k): + if k == 1: + yield (total,) + return + for first in range(1, total - k + 2): + for rest in _compositions(total - first, k - 1): + yield (first,) + rest diff --git a/root/wander-train/code_organ_v3.py b/root/wander-train/code_organ_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..3052c053f987c13898d6e908ff239c1d3c6c1a5a --- /dev/null +++ b/root/wander-train/code_organ_v3.py @@ -0,0 +1,488 @@ +"""code_organ_v3 - v2 (admitted in w482, immutable) + cover mode for conversation (coverage selection at any size +with stored programs allowed). v2: v1 (MBPP-frozen b2e52a1d) + STRICT mode for a single example: every move named by the +request. Programs as specified free moves over a verified function library, checked by execution; every +verified program joins the library (the stored final forms - continuous learning without retraining). + +REQUEST description + input/output examples (asserts) -> function name, argument values and types, expected outputs, + constants the description names ("at least 4 characters", "the character 'l'") +MOVES typed library functions. The TYPE SYSTEM is the specified structure: a function applies only where its + argument types connect; within that the choice is free. The description's words rank the moves. +SEARCH bottom-up enumeration with OBSERVATIONAL EQUIVALENCE: programs that behave identically on the examples are + kept once, so compositions of 3-4 moves stay tractable. +CHECK the rendered source is executed against every example; nothing is released that did not run and pass. +STORE a verified program becomes a library function (its description words are its keywords, its signature its + type); later requests compose it - measured as the library-growth curve. +Only library templates are ever evaluated; no text from the request is executed.""" +from __future__ import annotations + +import ast +import itertools +import json +import math +import re +from collections import Counter + +# ----------------------------------------------------------------------------------------------------------- types +def tyof(v): + if isinstance(v, bool): + return "bool" + if isinstance(v, int): + return "int" + if isinstance(v, float): + return "float" + if isinstance(v, str): + return "str" + if isinstance(v, list): + return "list" + if isinstance(v, tuple): + return "tuple" + if isinstance(v, dict): + return "dict" + if isinstance(v, set): + return "set" + return "any" + + +GROUP = {"num": {"int", "float"}, "seq": {"list", "tuple", "str"}, "coll": {"list", "tuple", "set"}, + "any": {"bool", "int", "float", "str", "list", "tuple", "dict", "set", "any"}} + + +def fits(t, want): + return t == want or t in GROUP.get(want, ()) + + +# ------------------------------------------------------------------------------------------------------ primitives +class Prim: + __slots__ = ("name", "ins", "out", "fn", "src", "kw") + + def __init__(self, name, ins, out, fn, src, kw): + self.name, self.ins, self.out, self.fn, self.src = name, ins, out, fn, src + self.kw = {w.lower().rstrip("s") for w in kw.split()} # v2: normalised exactly like the request words + + +def _big(x): + if isinstance(x, int) and abs(x) > 10 ** 7: + raise ValueError("too large") + return x + + +def _is_prime(n): + if n < 2: + return False + return all(n % d for d in range(2, int(n ** 0.5) + 1)) + + +def _uniq(xs): + out = [] + for x in xs: + if x not in out: + out.append(x) + return out + + +def _flatten(xs): + out = [] + for x in xs: + out += list(x) if isinstance(x, (list, tuple)) else [x] + return out + + +P_ = [ + # numbers + Prim("add", ("num", "num"), "num", lambda a, b: a + b, "({0} + {1})", "add sum plus total"), + Prim("sub", ("num", "num"), "num", lambda a, b: a - b, "({0} - {1})", "subtract difference minus"), + Prim("mul", ("num", "num"), "num", lambda a, b: _big(a * b), "({0} * {1})", "multiply product times"), + Prim("fdiv", ("num", "num"), "num", lambda a, b: a // b, "({0} // {1})", "divide quotient integer"), + Prim("div", ("num", "num"), "float", lambda a, b: a / b, "({0} / {1})", "divide ratio average"), + Prim("mod", ("num", "num"), "num", lambda a, b: a % b, "({0} % {1})", "remainder modulo divisible"), + Prim("pow", ("num", "num"), "num", lambda a, b: _big(a ** b) if b < 64 else 1 / 0, "({0} ** {1})", "power square cube exponent"), + Prim("sq", ("num",), "num", lambda a: a * a, "({0} ** 2)", "square squares"), + Prim("cube", ("num",), "num", lambda a: a ** 3, "({0} ** 3)", "cube cubes"), + Prim("abs", ("num",), "num", abs, "abs({0})", "absolute"), + Prim("neg", ("num",), "num", lambda a: -a, "(-{0})", "negative negate"), + Prim("half", ("num",), "float", lambda a: a / 2, "({0} / 2)", "half area triangle"), + Prim("sqrt", ("num",), "float", math.sqrt, "math.sqrt({0})", "root square"), + Prim("isqrt", ("int",), "int", math.isqrt, "math.isqrt({0})", "root square integer"), + Prim("fact", ("int",), "int", lambda n: math.factorial(n) if 0 <= n <= 30 else 1 / 0, "math.factorial({0})", "factorial"), + Prim("gcd", ("int", "int"), "int", math.gcd, "math.gcd({0}, {1})", "gcd greatest common divisor"), + Prim("lcm", ("int", "int"), "int", lambda a, b: a * b // math.gcd(a, b) if a and b else 0, "({0} * {1} // math.gcd({0}, {1}))", "lcm least common multiple"), + Prim("isprime", ("int",), "bool", _is_prime, "(({0}) > 1 and all(({0}) % d for d in range(2, int(({0}) ** 0.5) + 1)))", "prime"), + Prim("even", ("int",), "bool", lambda n: n % 2 == 0, "({0} % 2 == 0)", "even"), + Prim("odd", ("int",), "bool", lambda n: n % 2 == 1, "({0} % 2 == 1)", "odd"), + Prim("digits", ("int",), "list", lambda n: [int(c) for c in str(abs(n))], "[int(c) for c in str(abs({0}))]", "digits digit"), + Prim("revnum", ("int",), "int", lambda n: int(str(abs(n))[::-1]), "int(str(abs({0}))[::-1])", "reverse number"), + Prim("bin", ("int",), "str", lambda n: bin(n)[2:], "bin({0})[2:]", "binary"), + Prim("frombin", ("str",), "int", lambda s: int(s, 2), "int({0}, 2)", "binary decimal convert"), + Prim("popcount", ("int",), "int", lambda n: bin(n).count("1"), "bin({0}).count('1')", "set bits count ones"), + Prim("xor", ("int", "int"), "int", lambda a, b: a ^ b, "({0} ^ {1})", "xor bit differ"), + Prim("ispow2", ("int",), "bool", lambda n: n > 0 and n & (n - 1) == 0, "({0} > 0 and {0} & ({0} - 1) == 0)", "power two bit"), + Prim("eq", ("any", "any"), "bool", lambda a, b: a == b, "({0} == {1})", "equal same"), + Prim("lt", ("num", "num"), "bool", lambda a, b: a < b, "({0} < {1})", "less smaller"), + Prim("gt", ("num", "num"), "bool", lambda a, b: a > b, "({0} > {1})", "greater larger"), + Prim("not", ("bool",), "bool", lambda a: not a, "(not {0})", "not"), + Prim("and", ("bool", "bool"), "bool", lambda a, b: a and b, "({0} and {1})", "and both"), + Prim("or", ("bool", "bool"), "bool", lambda a, b: a or b, "({0} or {1})", "or either"), + Prim("maxn", ("num", "num"), "num", max, "max({0}, {1})", "maximum largest bigger"), + Prim("minn", ("num", "num"), "num", min, "min({0}, {1})", "minimum smallest"), + Prim("round2", ("num",), "float", lambda a: round(a, 2), "round({0}, 2)", "round"), + Prim("int", ("any",), "int", int, "int({0})", "integer convert"), + Prim("str", ("any",), "str", str, "str({0})", "string convert"), + Prim("float", ("num",), "float", float, "float({0})", "float"), + Prim("tri", ("int",), "int", lambda n: n * (n + 1) // 2, "({0} * ({0} + 1) // 2)", "triangular sum natural"), + Prim("pi_r2", ("num",), "float", lambda r: math.pi * r * r, "(math.pi * {0} * {0})", "area circle"), + Prim("circ", ("num",), "float", lambda r: 2 * math.pi * r, "(2 * math.pi * {0})", "circumference perimeter circle"), + # sequences + Prim("len", ("seq",), "int", len, "len({0})", "length count number size"), + Prim("sum", ("coll",), "num", lambda xs: sum(xs), "sum({0})", "sum total add"), + Prim("max", ("coll",), "any", max, "max({0})", "maximum largest biggest max longest"), + Prim("min", ("coll",), "any", min, "min({0})", "minimum smallest min"), + Prim("sorted", ("coll",), "list", sorted, "sorted({0})", "sort sorted ascending order"), + Prim("sorted_desc", ("coll",), "list", lambda xs: sorted(xs, reverse=True), "sorted({0}, reverse=True)", "sort descending largest"), + Prim("rev", ("seq",), "seq", lambda xs: xs[::-1], "{0}[::-1]", "reverse reversed"), + Prim("first", ("seq",), "any", lambda xs: xs[0], "{0}[0]", "first"), + Prim("last", ("seq",), "any", lambda xs: xs[-1], "{0}[-1]", "last"), + Prim("take", ("seq", "int"), "seq", lambda xs, n: xs[:n], "{0}[:{1}]", "first n smallest largest take"), + Prim("drop", ("seq", "int"), "seq", lambda xs, n: xs[n:], "{0}[{1}:]", "remove first skip"), + Prim("at", ("seq", "int"), "any", lambda xs, i: xs[i], "{0}[{1}]", "nth index element second third"), + Prim("tolist", ("any",), "list", list, "list({0})", "list convert"), + Prim("totuple", ("any",), "tuple", tuple, "tuple({0})", "tuple convert"), + Prim("toset", ("coll",), "set", set, "set({0})", "unique distinct set"), + Prim("uniq", ("coll",), "list", _uniq, "list(dict.fromkeys({0}))", "unique distinct duplicates remove"), + Prim("inter", ("coll", "coll"), "tuple", lambda a, b: tuple(sorted(set(a) & set(b))), "tuple(sorted(set({0}) & set({1})))", "common shared similar intersection"), + Prim("union", ("coll", "coll"), "tuple", lambda a, b: tuple(sorted(set(a) | set(b))), "tuple(sorted(set({0}) | set({1})))", "union combine"), + Prim("diff", ("coll", "coll"), "list", lambda a, b: [x for x in a if x not in b], "[x for x in {0} if x not in {1}]", "difference not in"), + Prim("concat", ("seq", "seq"), "seq", lambda a, b: a + b, "({0} + {1})", "concatenate join append combine merge"), + Prim("count", ("seq", "any"), "int", lambda xs, x: xs.count(x), "{0}.count({1})", "count occurrences frequency"), + Prim("index", ("seq", "any"), "int", lambda xs, x: xs.index(x), "{0}.index({1})", "index position find"), + Prim("contains", ("any", "seq"), "bool", lambda x, xs: x in xs, "({0} in {1})", "contains present exists check"), + Prim("allsame", ("coll",), "bool", lambda xs: len(set(xs)) <= 1, "(len(set({0})) <= 1)", "same equal all identical"), + Prim("alluniq", ("coll",), "bool", lambda xs: len(set(xs)) == len(xs), "(len(set({0})) == len(list({0})))", "unique distinct duplicate"), + Prim("issorted", ("coll",), "bool", lambda xs: list(xs) == sorted(xs), "(list({0}) == sorted({0}))", "sorted order check"), + Prim("prod", ("coll",), "num", lambda xs: _big(math.prod(xs)), "math.prod({0})", "product multiply"), + Prim("mean", ("coll",), "float", lambda xs: sum(xs) / len(xs), "(sum({0}) / len({0}))", "average mean"), + Prim("map_sq", ("coll",), "list", lambda xs: [x * x for x in xs], "[x * x for x in {0}]", "square squares each"), + Prim("map_cube", ("coll",), "list", lambda xs: [x ** 3 for x in xs], "[x ** 3 for x in {0}]", "cube cubes each"), + Prim("map_neg", ("coll",), "list", lambda xs: [-x for x in xs], "[-x for x in {0}]", "negative"), + Prim("map_abs", ("coll",), "list", lambda xs: [abs(x) for x in xs], "[abs(x) for x in {0}]", "absolute"), + Prim("map_str", ("coll",), "list", lambda xs: [str(x) for x in xs], "[str(x) for x in {0}]", "string convert"), + Prim("map_int", ("coll",), "list", lambda xs: [int(x) for x in xs], "[int(x) for x in {0}]", "integer convert"), + Prim("map_len", ("coll",), "list", lambda xs: [len(x) for x in xs], "[len(x) for x in {0}]", "length each"), + Prim("map_first", ("coll",), "list", lambda xs: [x[0] for x in xs], "[x[0] for x in {0}]", "first each"), + Prim("map_last", ("coll",), "list", lambda xs: [x[-1] for x in xs], "[x[-1] for x in {0}]", "last each"), + Prim("map_sum", ("coll",), "list", lambda xs: [sum(x) for x in xs], "[sum(x) for x in {0}]", "sum each"), + Prim("map_add", ("coll", "num"), "list", lambda xs, k: [x + k for x in xs], "[x + {1} for x in {0}]", "add each"), + Prim("map_mul", ("coll", "num"), "list", lambda xs, k: [x * k for x in xs], "[x * {1} for x in {0}]", "multiply each double doubled twice"), + Prim("zipadd", ("coll", "coll"), "list", lambda a, b: [x + y for x, y in zip(a, b)], "[x + y for x, y in zip({0}, {1})]", "add two lists elementwise"), + Prim("zipmul", ("coll", "coll"), "list", lambda a, b: [x * y for x, y in zip(a, b)], "[x * y for x, y in zip({0}, {1})]", "multiply two lists"), + Prim("zipsub", ("coll", "coll"), "list", lambda a, b: [x - y for x, y in zip(a, b)], "[x - y for x, y in zip({0}, {1})]", "subtract two lists"), + Prim("zipdiv", ("coll", "coll"), "list", lambda a, b: [x / y for x, y in zip(a, b)], "[x / y for x, y in zip({0}, {1})]", "divide two lists"), + Prim("zip", ("coll", "coll"), "list", lambda a, b: list(zip(a, b)), "list(zip({0}, {1}))", "pair zip"), + Prim("f_even", ("coll",), "list", lambda xs: [x for x in xs if x % 2 == 0], "[x for x in {0} if x % 2 == 0]", "even"), + Prim("f_odd", ("coll",), "list", lambda xs: [x for x in xs if x % 2 == 1], "[x for x in {0} if x % 2 == 1]", "odd"), + Prim("f_pos", ("coll",), "list", lambda xs: [x for x in xs if x > 0], "[x for x in {0} if x > 0]", "positive"), + Prim("f_neg", ("coll",), "list", lambda xs: [x for x in xs if x < 0], "[x for x in {0} if x < 0]", "negative"), + Prim("f_gt", ("coll", "num"), "list", lambda xs, k: [x for x in xs if x > k], "[x for x in {0} if x > {1}]", "greater than"), + Prim("f_lenge", ("coll", "int"), "list", lambda xs, k: [x for x in xs if len(x) >= k], "[x for x in {0} if len(x) >= {1}]", "at least long characters"), + Prim("f_neq", ("coll", "any"), "list", lambda xs, k: [x for x in xs if x != k], "[x for x in {0} if x != {1}]", "remove all"), + Prim("cumsum", ("coll",), "list", lambda xs: list(itertools.accumulate(xs)), "list(itertools.accumulate({0}))", "cumulative running sum"), + Prim("flatten", ("coll",), "list", _flatten, "[y for x in {0} for y in (x if isinstance(x, (list, tuple)) else [x])]", "flatten nested"), + Prim("mostcommon", ("coll",), "any", lambda xs: Counter(xs).most_common(1)[0][0], "collections.Counter({0}).most_common(1)[0][0]", "most common frequent maximum occurrences"), + Prim("freq", ("coll",), "dict", lambda xs: dict(Counter(xs)), "dict(collections.Counter({0}))", "frequency count each"), + Prim("sortkey1", ("coll",), "list", lambda xs: sorted(xs, key=lambda t: t[1]), "sorted({0}, key=lambda t: t[1])", "sort second element"), + Prim("maxkey_len", ("coll",), "any", lambda xs: max(xs, key=len), "max({0}, key=len)", "longest maximum length"), + Prim("minkey_len", ("coll",), "any", lambda xs: min(xs, key=len), "min({0}, key=len)", "shortest minimum length"), + Prim("range1", ("int",), "list", lambda n: list(range(1, n + 1)) if n <= 10 ** 5 else 1 / 0, "list(range(1, {0} + 1))", "first n natural numbers range"), + Prim("range0", ("int",), "list", lambda n: list(range(n)) if n <= 10 ** 5 else 1 / 0, "list(range({0}))", "range"), + # strings + Prim("upper", ("str",), "str", str.upper, "{0}.upper()", "upper uppercase capital"), + Prim("lower", ("str",), "str", str.lower, "{0}.lower()", "lower lowercase"), + Prim("title", ("str",), "str", str.title, "{0}.title()", "title capitalize first letter each word"), + Prim("capitalize", ("str",), "str", str.capitalize, "{0}.capitalize()", "capitalize"), + Prim("swapcase", ("str",), "str", str.swapcase, "{0}.swapcase()", "toggle swap case"), + Prim("strip", ("str",), "str", str.strip, "{0}.strip()", "strip trim whitespace spaces"), + Prim("nospace", ("str",), "str", lambda s: s.replace(" ", ""), "{0}.replace(' ', '')", "remove spaces whitespace"), + Prim("words", ("str",), "list", str.split, "{0}.split()", "words split"), + Prim("splitby", ("str", "str"), "list", lambda s, d: s.split(d), "{0}.split({1})", "split delimiter"), + Prim("joinsp", ("coll",), "str", lambda xs: " ".join(xs), "' '.join({0})", "join words"), + Prim("join", ("coll",), "str", lambda xs: "".join(xs), "''.join({0})", "join concatenate characters"), + Prim("replace", ("str", "str", "str"), "str", lambda s, a, b: s.replace(a, b), "{0}.replace({1}, {2})", "replace"), + Prim("remove", ("str", "str"), "str", lambda s, a: s.replace(a, ""), "{0}.replace({1}, '')", "remove delete"), + Prim("remove1", ("str", "str"), "str", lambda s, a: s.replace(a, "", 1), "{0}.replace({1}, '', 1)", "remove first occurrence"), + Prim("removelast", ("str", "str"), "str", lambda s, a: (s[:s.rfind(a)] + s[s.rfind(a) + 1:]) if a in s else s, "(({0}[:{0}.rfind({1})] + {0}[{0}.rfind({1}) + 1:]) if {1} in {0} else {0})", "remove last occurrence"), + Prim("chars", ("str",), "list", list, "list({0})", "characters chars"), + Prim("isdigit", ("str",), "bool", str.isdigit, "{0}.isdigit()", "digit numeric integer"), + Prim("isalpha", ("str",), "bool", str.isalpha, "{0}.isalpha()", "alphabet letters"), + Prim("isupper", ("str",), "bool", str.isupper, "{0}.isupper()", "upper"), + Prim("islower", ("str",), "bool", str.islower, "{0}.islower()", "lower"), + Prim("startswith", ("str", "str"), "bool", str.startswith, "{0}.startswith({1})", "starts begins"), + Prim("endswith", ("str", "str"), "bool", str.endswith, "{0}.endswith({1})", "ends"), + Prim("find", ("str", "str"), "int", str.find, "{0}.find({1})", "find position index occurrence"), + Prim("f_upper", ("str",), "str", lambda s: "".join(c for c in s if c.isupper()), "''.join(c for c in {0} if c.isupper())", "uppercase characters extract"), + Prim("f_lower", ("str",), "str", lambda s: "".join(c for c in s if c.islower()), "''.join(c for c in {0} if c.islower())", "lowercase characters remove"), + Prim("f_digit", ("str",), "str", lambda s: "".join(c for c in s if c.isdigit()), "''.join(c for c in {0} if c.isdigit())", "digits numbers extract"), + Prim("f_alpha", ("str",), "str", lambda s: "".join(c for c in s if c.isalpha()), "''.join(c for c in {0} if c.isalpha())", "letters remove non alphabet"), + Prim("f_alnum", ("str",), "str", lambda s: "".join(c for c in s if c.isalnum()), "''.join(c for c in {0} if c.isalnum())", "alphanumeric remove special characters"), + Prim("vowels", ("str",), "int", lambda s: sum(c in "aeiouAEIOU" for c in s), "sum(c in 'aeiouAEIOU' for c in {0})", "vowels count"), + Prim("ispal", ("seq",), "bool", lambda s: s == s[::-1], "({0} == {0}[::-1])", "palindrome"), + Prim("sortchars", ("str",), "str", lambda s: "".join(sorted(s)), "''.join(sorted({0}))", "sort characters"), + Prim("rev_words", ("str",), "str", lambda s: " ".join(s.split()[::-1]), "' '.join({0}.split()[::-1])", "reverse words"), + Prim("ord", ("str",), "int", lambda s: ord(s) if len(s) == 1 else 1 / 0, "ord({0})", "ascii value character"), + Prim("chr", ("int",), "str", lambda n: chr(n) if 0 <= n < 1114111 else 1 / 0, "chr({0})", "character ascii"), + Prim("strlen_words", ("str",), "list", lambda s: [len(w) for w in s.split()], "[len(w) for w in {0}.split()]", "length words"), + Prim("firstchar", ("str",), "str", lambda s: s[0], "{0}[0]", "first character"), + Prim("lastchar", ("str",), "str", lambda s: s[-1], "{0}[-1]", "last character"), + Prim("upper_first_last", ("str",), "str", lambda s: " ".join((w[0].upper() + w[1:-1] + w[-1].upper()) if len(w) > 1 else w.upper() for w in s.split()), "' '.join((w[0].upper() + w[1:-1] + w[-1].upper()) if len(w) > 1 else w.upper() for w in {0}.split())", "capitalize first last letters each word"), +] +# v2 library entries added from the gate panel's FAILURE CLASSES (general-purpose operations; checked on MBPP dev) +P_ += [ + Prim("c2f", ("num",), "float", lambda c: c * 9 / 5 + 32, "({0} * 9 / 5 + 32)", "celsius fahrenheit convert temperature"), + Prim("f2c", ("num",), "float", lambda f: (f - 32) * 5 / 9, "(({0} - 32) * 5 / 9)", "fahrenheit celsius convert temperature"), + Prim("isleap", ("int",), "bool", lambda y: y % 4 == 0 and (y % 100 != 0 or y % 400 == 0), + "({0} % 4 == 0 and ({0} % 100 != 0 or {0} % 400 == 0))", "leap year"), + Prim("joincomma", ("coll",), "str", lambda xs: ",".join(xs), "','.join({0})", "join comma separated"), + Prim("nwords", ("str",), "int", lambda s: len(s.split()), "len({0}.split())", "count words number"), +] +IMPORTS = "import math\nimport itertools\nimport collections\n" + + +# --------------------------------------------------------------------------------------------------------- request +def parse_tests(tests, setup=""): + """'assert f(a, b) == v' -> (fname, [(args, expected)], wrapper-src or None)""" + fname, cases, wrap = None, [], None + for t in tests: + try: + node = ast.parse(t.strip()).body[0] + except SyntaxError: + return None + if not isinstance(node, ast.Assert): + return None + test = node.test + if isinstance(test, ast.Call) or (isinstance(test, ast.UnaryOp) and isinstance(test.operand, ast.Call)): + neg = isinstance(test, ast.UnaryOp) + call = test.operand if neg else test + exp = not neg + lhs = call + elif isinstance(test, ast.Compare) and len(test.ops) == 1 and isinstance(test.ops[0], ast.Eq): + lhs = test.left + try: + exp = ast.literal_eval(test.comparators[0]) + except Exception: # noqa: BLE001 + return None + else: + return None + call, w = lhs, None + if isinstance(lhs, ast.Call) and isinstance(lhs.func, ast.Name) and lhs.func.id in ("set", "sorted", "list", "tuple", "round", "abs") and lhs.args and isinstance(lhs.args[0], ast.Call): + w, call = lhs.func.id, lhs.args[0] + if not (isinstance(call, ast.Call) and isinstance(call.func, ast.Name)): + return None + try: + args = [ast.literal_eval(a) for a in call.args] + except Exception: # noqa: BLE001 + return None + if fname and call.func.id != fname: + return None + fname, wrap = call.func.id, w or wrap + cases.append((args, exp)) + return (fname, cases, wrap) if fname and cases else None + + +WRAP = {None: lambda v: v, "set": set, "sorted": sorted, "list": list, "tuple": tuple, "round": round, "abs": abs} + + +ORDINAL = {"first": 0, "second": 1, "third": 2, "fourth": 3, "last": -1, "double": 2, "doubled": 2, "twice": 2, + "triple": 3, "tripled": 3, "half": 2, "halve": 2, "halved": 2} + + +def consts_of(desc, cases, strict=False): + """constants a program may use. STRICT (one example): only what the request itself names - its numbers, its + quoted characters and its ordinals (first=0, second=1, last=-1) - so no positional coincidence is reachable""" + text = re.split(r"\bso\b", desc or "")[0] if strict else (desc or "") + named = [int(n) for n in re.findall(r"\b(\d{1,4})\b", text)] + named += re.findall(r"'([^']{1,3})'", text) + named += [v for w, v in ORDINAL.items() if re.search(r"\b%s\b" % w, text, re.I)] + out = named if strict else [0, 1, 2, -1, "", " "] + named + return list(dict.fromkeys(out)) + + +GENERIC = {"write", "function", "python", "find", "given", "number", "numbers", "integer", "list", "string", "element", + "elements", "value", "values", "check", "whether", "two", "the", "and", "from", "all", "using", "which", "that", + "with", "into", "for", "each", "array", "tuple", "item", "items", "get", "given", "not", "out", "sequence"} + + +def words_of(desc): + return {w.lower().rstrip("s") for w in re.findall(r"[A-Za-z]+", desc or "") + if len(w) >= 3 and w.lower() not in GENERIC and w.lower().rstrip("s") not in GENERIC} + + +# ----------------------------------------------------------------------------------------------------------- search +class CodeOrgan: + def __init__(self, library=None, max_size=4, cap=60000): + self.prims = list(P_) + list(library or []) + self.max_size, self.cap = max_size, cap + self.why = "" + + def learn(self, name, prog, arity_types, out_t, desc): + """STORE: a verified program becomes a library function; later requests compose it""" + fnc = prog["fn"] + src = "%s(%s)" % (name, ", ".join("{%d}" % i for i in range(len(arity_types)))) + p = Prim("lib_" + name, tuple(arity_types), out_t, lambda *vals: fnc(vals), src, " ".join(words_of(desc))) + self.prims.append(p) + self.library_src = getattr(self, "library_src", {}) + self.library_src[name] = prog["def"].replace(IMPORTS, "") + return p + + def solve(self, desc, tests, setup="", visible=None, strict=False, cover=False, allow_lib=True): + """strict: request-named constants only + coverage selection (one example); cover: coverage selection at any + size (conversation); allow_lib: stored programs may compose (needs >= 2 examples)""" + cover = cover or strict + self.why = "" + pt = parse_tests(tests, setup) + if not pt: + self.why = "the examples are not simple asserts" + return None + fname, cases, wrap = pt + vis = cases if visible is None else cases[:visible] + if not vis: + return None + W = WRAP[wrap] + nargs = len(vis[0][0]) + if any(len(a) != nargs for a, _ in cases): + self.why = "varying arity" + return None + target = [W(e) if wrap else e for _, e in vis] + kw = words_of(desc) + self._req_kw, self._cover, self._n_visible = kw, None, len(vis) + # the request SPECIFIES the moves: rank by the description's words; keep the whole library but try relevant first + # a stored program is a move only with >= 2 examples, and never inside its own definition; with ONE example + # (strict) only the base library composes - forms verified on one example are too weak to build on from one more + pool = [p for p in self.prims if p.name != "lib_" + fname and not ((strict or not allow_lib) and p.name.startswith("lib_"))] + prims = sorted(pool, key=lambda p: -len(p.kw & kw)) + # terminals: arguments and constants; a term = (src, type, outputs-tuple, fn) + terms_by_size = {1: []} + seen = set() + + def add(size, src, outs, fnc, used=frozenset()): + key = repr(outs) + try: + match = [W(o) if wrap else o for o in outs] == target + except Exception: # noqa: BLE001 + match = False + if match: + # the request SPECIFIES the moves: a fit made only of moves the description never names is a + # coincidence of three examples, not an answer - it is not accepted and does not claim the signature. + # A fit must also READ its inputs: every argument is used, except an 'n' that only restates a length. + reads = all(argn[i] in re.findall(r"a\d+", src) or restates_len[i] for i in range(nargs)) + # STRICT (one example): EVERY operation must be named by the request - the description specifies the + # program and the single example only checks it + named = all(pr.kw & kw for pr in used) if strict else any(pr.kw & kw for pr in used) + if used and reads and named: + return (src, fnc) + self.coincidences = getattr(self, "coincidences", 0) + 1 + return None + if key in seen or len(key) > 1500 or len(seen) > self.cap: + return None # bounded memory: huge values and past-budget terms are not kept + seen.add(key) + ty = tyof(outs[0]) + if any(tyof(o) != ty for o in outs): + ty = "any" + terms_by_size.setdefault(size, []).append((src, ty, outs, fnc, used)) + return None + + argn = ["a%d" % i for i in range(nargs)] + restates_len = [all(isinstance(a[i], int) and any(isinstance(a[j], (list, tuple, str)) and len(a[j]) == a[i] + for j in range(nargs) if j != i) for a, _ in vis) + for i in range(nargs)] + for i in range(nargs): + hit = add(1, argn[i], tuple(a[i] for a, _ in vis), (lambda i: lambda xs: xs[i])(i)) + if hit: + return self._emit(fname, argn, hit, cases, wrap, desc) + for c in consts_of(desc, vis, strict): + add(1, repr(c), tuple(c for _ in vis), (lambda c: lambda xs: c)(c)) + n_terms = 0 + best = None + first_hit_size = None + for size in range(2, self.max_size + 1): + if cover and best and first_hit_size is None: + first_hit_size = size - 1 + if cover and first_hit_size is not None and size > first_hit_size + 1: + break # v3: look at most ONE size beyond the first fit (bounded time) + for p in prims: + k = len(p.ins) + # child sizes summing to size-1 + for parts in _compositions(size - 1, k): + pools = [terms_by_size.get(s, []) for s in parts] + pools = [[t for t in pool if fits(t[1], want) or t[1] == "any"] for pool, want in zip(pools, p.ins)] + if any(not pool for pool in pools): + continue + for combo in itertools.product(*pools): + n_terms += 1 + if n_terms > self.cap: + if cover and best: # v3: a found fit is never discarded by the budget + return self._emit(fname, argn, best[1], cases, wrap, desc) + self.why = "search budget exhausted" + return None + try: + outs = tuple(p.fn(*[t[2][j] for t in combo]) for j in range(len(vis))) + except Exception: # noqa: BLE001 + continue + if any(_oversize(o) for o in outs): + continue # bounded memory: never represent a huge intermediate value + src = p.src.format(*[t[0] for t in combo]) + fnc = (lambda p, combo: lambda xs: p.fn(*[t[3](xs) for t in combo]))(p, combo) + used = frozenset([p]).union(*[t[4] for t in combo]) + hit = add(size, src, outs, fnc, used) + if hit: + if not cover: + return self._emit(fname, argn, hit, cases, wrap, desc) + # COVERAGE: how much of what the request says the program's moves account for, jointly + named = len(set().union(*[pr.kw for pr in used]) & kw) + if best is None or named > best[0]: + best = (named, hit) + self._cover = named + if cover and best: # v2 STRICT: the fit covering most of the request, any size + return self._emit(fname, argn, best[1], cases, wrap, desc) + self.why = "no program of size <= %d fits the examples" % self.max_size + return None + + def _emit(self, fname, argn, hit, cases, wrap, desc): + src, fnc = hit + lib = getattr(self, "library_src", {}) + used = [n for n in lib if re.search(r"\b%s\(" % re.escape(n), src)] + deps = "".join(lib[n] for n in used) # stored forms this program composes + body = "%s%sdef %s(%s):\n return %s\n" % (IMPORTS, deps, fname, ", ".join(argn), src) + # CHECK: execute the RENDERED source (not the in-memory closure) against every example given + ns = {} + exec(compile(body, "", "exec"), ns) # noqa: S102 - composed only of library templates + W = WRAP[wrap] + passed = 0 + for a, e in cases: + try: + out = ns[fname](*a) + ok = (W(out) if wrap else out) == e + except Exception: # noqa: BLE001 + ok = False + passed += ok + return {"def": body, "fname": fname, "expr": src, "passed": passed, "cases": len(cases), "fn": fnc, + "arity": len(argn), "reused": used, "cover": getattr(self, "_cover", None), + "req_words": len(getattr(self, "_req_kw", ()) or ()), "n_visible": getattr(self, "_n_visible", None)} + + +def _oversize(o): + if isinstance(o, bool): + return False + if isinstance(o, int): + return abs(o) > 10 ** 12 + if isinstance(o, str): + return len(o) > 400 + if isinstance(o, (list, tuple, set, dict)): + return len(o) > 200 or any(_oversize(x) for x in list(o)[:50]) + return False + + +def _compositions(total, k): + if k == 1: + yield (total,) + return + for first in range(1, total - k + 2): + for rest in _compositions(total - first, k - 1): + yield (first,) + rest diff --git a/root/wander-train/exact_router_v1.py b/root/wander-train/exact_router_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..6f4e7e61678f7cba98cfe17a38848f008ec2c416 --- /dev/null +++ b/root/wander-train/exact_router_v1.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""exact_router_v1 - Oracle7's EXACT relation router (runtime form). No similarity, no learned weights: a question is answered +only when an ADMITTED pattern names the relation it asks and the linked entities agree on one stored value. + +Patterns (router_mine_v3, 2026-10-06): (asked noun or wh-word, one content word) -> Wikidata property, mined over 124,000 +TriviaQA train questions; admitted when the answer given through that property is right >= 90% of the time on BOTH md5 +halves with >= 8 firings in each (32 patterns, 13 properties). Never-read holdout (TriviaQA train offset 82,000): 33 right, +1 wrong (the wrong one was a Wikidata unknown-value blank node - now refused, rule 3 below). +Answer: entity_linker_v2 top span -> its candidates (<= 3) -> every candidate carrying the routed property must agree on +ONE value. Refused: (1) yes/no questions; (2) a hidden hop - an unnamed intermediate between the named entity and the +answer ("an artist who also directed", "the brother-in-law of X"), the guard measured 2026-10-06; (3) blank-node values; +(4) every matching pattern must name the SAME property. +use: answer(question) -> {"answer", "property", "subject_qid", "pattern"} or None""" +import json +import re + +import entity_linker_v2 as L +import oracle_relstore_v1 as RS + +PATTERNS = __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/relstore/router_patterns_v4.json" +STORE = __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/relstore/relations_v2.sqlite3" +_PATS, _RS = None, None +norm = L.norm +HEAD = re.compile(r"\b(?:which|what)\s+((?:[a-z]+\s+){0,2}?[a-z]+)(?:\s+(?:is|was|are|were|did|does|do|has|had|in|of|for|from|" + r"to|by|named|called|founded|created|directed|wrote|won|became)\b|\s*,)", re.I) +WH = re.compile(r"\b(who|whom|whose|where|when|which|what|how)\b", re.I) +POLAR = re.compile(r"^\s*(is|are|was|were|do|does|did|has|have|had|can|could|will|would|should)\b", re.I) +HOP = re.compile(r"\b(a|an|the)\s+([\w'–-]+\s+){0,3}(who|whom|whose|which|that)\b(?!\s*\?)" + r"|\b(a|an|the)\s+([\w'–-]+\s+){0,5}(created|directed|written|founded|coached|owned|produced|hosted|led|played|named)\s+by\b" + r"|\bthe\s+(illustrator|author|director|founder|creator|wife|husband|father|mother|son|daughter|brother|sister|" + r"brother-in-law|sister-in-law|father-in-law|mother-in-law|son-in-law|daughter-in-law|nephew|niece|uncle|aunt|" + r"owner|composer|producer|writer|singer|star|team|club|band|company|capital|leader|coach|manager)\s+of\b", re.I) + + +def _load(): + global _PATS, _RS + if _PATS is None: + _PATS = {tuple(k.split("|", 1)): v[0] for k, v in json.load(open(PATTERNS)).items()} + _RS = RS.RelStore() if STORE is None else RS.RelStore(STORE) + return _PATS, _RS + + +def features(q, span): + m = HEAD.search(q) + head = norm(m.group(1)).split()[-1] if m else None + w = WH.search(q) + key = head or (w.group(1).lower() if w else "-") + ent = set(norm(span).split()) + words = {x for x in norm(q).split() if x not in L.STOP and x not in ent and not x.isdigit() and len(x) >= 3} + return {(key, x) for x in words} | {(key, "*")} + + +DOMINANCE = 10 # a name whose top sense is >= 10x more linked refers to it + + +def hop(q, span): + """a hidden hop - but "the capital of France" is no hop when France is the named entity right after "of" """ + for m in HOP.finditer(q): + if m.group(0).lower().endswith(" of"): + after = norm(q[m.end():]) + if after.startswith(norm(span)) or after.startswith("the " + norm(span)): + continue + return True + return False + + +# the store holds CURRENT values: a past-time qualifier on a relation that changes over time is not answerable from it +TIME_VARIANT = {36, 35, 6, 54, 159, 27, 17, 131, 286, 488, 1037, 108, 39, 463, 102, 26} +PAST = re.compile(r"\b(between|until|till|before|after|during|in|from|by)\s+(the\s+)?(\d{3,4}s?|\d{1,2}(st|nd|rd|th)\s+century)\b" + r"|\b(formerly|former|originally|once|previously|ancient|old)\b", re.I) +QUOTED = re.compile(r"[‘“\"']([^’”\"']{3,80})[’”\"']") + + +def subject_ok(q, span): + """the linked span must be the question's subject: a quoted title must BE the span; a later "of " means the + subject is further along the "of" chain ("the capital of the U.S. state of Connecticut")""" + s = q.strip() + if len(s) > 2 and s[0] == '"' and s[-1] == '"': + s = s[1:-1] # CSV-wrapped question: "Who ... ""Citizen Kane""?" + m = QUOTED.search(s.replace('""', '"')) + if m and norm(m.group(1)) != norm(span): + return False + i = q.find(span) + if i >= 0 and re.search(r"\bof\s+(the\s+)?[A-Z]", q[i + len(span):]): + return False + return True + + +def answer(q): + pats, rs = _load() + q = str(q) + if POLAR.search(q): + return None + sp = L.link(q, 3) + if not sp: + return None + span, cs = sp[0] + if hop(q, span) or not subject_ok(q, span): + return None + if DOMINANCE and len(cs) > 1 and cs[0][1] >= DOMINANCE * max(1, max(c[1] for c in cs[1:])): + cs = cs[:1] + matched = {f: pats[f] for f in features(q, span) if f in pats} + props = set(matched.values()) + if len(props) != 1: + return None + p = props.pop() + if p in TIME_VARIANT and PAST.search(q): + return None + sets, subj = [], None + for qid, _, _ in cs[:3]: + vals = {rs.label(o) if o else lit for pp, o, lit in rs.props_of(qid) if pp == p} + vals = {v for v in vals if v and ".well-known/genid" not in v} + if vals: + sets.append(frozenset(vals)) + subj = subj or qid + if not sets or len(set(sets)) != 1 or len(sets[0]) != 1: + return None + aw = [w for w in norm(next(iter(sets[0]))).split() if w not in L.STOP] + if aw and all(w in set(norm(q).split()) for w in aw): + return None # the answer restates the question: not an answer + return {"answer": _show(next(iter(sets[0])), q), "property": p, "subject_qid": subj, + "pattern": "|".join(sorted(matched)[0])} + + +MONTHS = ("January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December") + + +def _show(v, q): + """a stored time "1034-03-27T00:00:00Z" reads as "1034" when the year is asked, else "27 March 1034"; BCE years keep + their sign as "BC" """ + m = re.match(r"^([+-]?)(\d{1,4})-(\d\d)-(\d\d)T", v) + if not m: + return v + y = str(int(m.group(2))) + (" BC" if m.group(1) == "-" else "") + if re.search(r"\byear\b", q, re.I) or m.group(3) == "00" or (m.group(3) == "01" and m.group(4) == "01"): + return y + return "%d %s %s" % (int(m.group(4)), MONTHS[int(m.group(3)) - 1], y) + + +if __name__ == "__main__": + import sys + for q in sys.argv[1:]: + print(q, "->", answer(q)) diff --git a/root/wander-train/general_reader_v1.py b/root/wander-train/general_reader_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..e13aecbbc338b76d6b79328686b2c702ef2a00b5 --- /dev/null +++ b/root/wander-train/general_reader_v1.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""general_reader_v1 - a relation reader that does not depend on how a question is phrased (Box C). +For a question the release declines: find the entities it names (exact Wikipedia titles), read every fact one step from them in +the Wikidata relation store, and answer ONLY when all three agree: + (1) CUE - the relation is named by the question's own words (property label stems, or a measured router cue pair) + (2) TYPE - the answer is the kind of thing asked for (who -> a person; where -> a place; "which comedian" -> a comedian) + (3) UNIQUE- exactly one candidate survives (two different answers -> decline) +The answer is never an entity the question itself names. Every answer carries its receipt (subject, property, value, ids). +usage (dev measurement): general_reader_v1.py eval | general_reader_v1.py ask "" """ +import json +import re +import sqlite3 +import sys +import unicodedata + +STORE = __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/relstore/relations_v2.sqlite3" +db = sqlite3.connect("file:%s?mode=ro&immutable=1" % STORE, uri=True, check_same_thread=False) + +STOP = set("a an the of in on at to for by with from and or is was were are be been being who whom whose what which when where why how " + "did does do has have had this that these those it its he she they his her their as into than then there here one".split()) +AUX = set("is was were are did does do has have had can could will would should may might".split()) +HUMAN_Q = ("who", "whom", "whose") +PLACE_WORDS = set("city town village country state province region county island capital place municipality commune borough " + "district settlement port river mountain lake sea ocean continent area".split()) +GENERIC_TYPE = set("one name person people thing year type kind part title area word term number".split()) +# qualifiers the reader cannot evaluate from one fact: order, extremes, time, fiction or a work's own world +QUALIFIER = re.compile(r"\b(first|last|only|second|third|fourth|fifth|oldest|youngest|largest|smallest|biggest|longest|shortest|" + r"highest|lowest|tallest|most|least|best|worst|original|former|current|currently|previous|next|latest|earliest|" + r"famous|famously|nicknamed|known as|better known|called|real name|in the (?:bible|film|movie|novel|book|play|" + r"series|show|song|opera|musical|game|tv|story|poem|legend)|in (?:greek|roman|norse) mythology|" + r"(?:1[0-9]|20)[0-9]{2}|century|decade|year|when|during|at the time|outbreak|battle|war|preceded|succeeded|before|after|" + r"between|same|other|also|both|nickname|setting|set in|according to)\b", re.I) + +PROP, PROPNAMES = {}, {} +for p, lab in db.execute("SELECT p, label FROM prop_src"): + if lab: + PROP[p] = lab +for line in open(__import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/relstore/prop_labels.tsv", encoding="utf-8"): + m = re.match(r'\t"(.*)"@en', line.strip()) + if m: + p = int(m.group(1)) + PROPNAMES.setdefault(p, []).append(m.group(2)) # the file holds the label AND its aliases, one per line + PROP.setdefault(p, m.group(2)) +# everyday words for common relations (question wording differs from Wikidata's label); measured on dev like everything else +ALIASES = {26: "married marry wife husband spouse", 25: "mother", 22: "father", 3373: "sister brother sibling", 40: "son daughter child", + 50: "writer author novelist", 57: "directed director", 86: "composed composer music", 170: "created creator", + 112: "founded founder", 36: "capital", 19: "born birthplace", 20: "died death", 569: "born", 570: "died", + 175: "singer performer", 161: "starred", 69: "studied attended educated", 108: "worked employer", + 54: "played team", 166: "won awarded award", 131: "located situated", 138: "named", 37: "language spoken official", + 38: "currency", 6: "prime minister mayor premier", 35: "president king queen monarch emperor", 27: "nationality citizen", + 800: "known work", 61: "discovered discoverer", 397: "orbits orbit", 30: "continent", 140: "religion", 102: "party", + 127: "owned owner owns", 176: "made built manufactured manufacturer", 84: "designed architect", 674: "characters character", + 1441: "appears appear", 136: "genre", 264: "label", 98: "edited editor", 162: "produced producer", 58: "screenplay screenwriter", + 710: "participant participated", 1346: "won winner", 17: "country", 495: "country", 159: "headquarters based", 740: "formed", + 571: "founded established formed", 576: "dissolved abolished", 1037: "director manager", 488: "chairman chairperson", + 169: "chief executive ceo", 286: "head coach manager", 451: "partner", 1038: "relative", 1327: "partner", 413: "position played", + 118: "league", 641: "sport", 115: "home stadium ground venue", 361: "part", 150: "contains", 206: "located", 1376: "capital"} +ALIAS_STEMS = {} +try: + ALLOW = set(json.load(open(__import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/relstore/reader_prop_allowlist.json"))) # relations the admitted relation specialists read +except Exception: # noqa: BLE001 + ALLOW = set() +ALLOW -= {102, 1082} # party membership changes; population is a number +SKIP_PROPS = {21, 971, 735, 734, 1343, 910, 4224, 1889, 31, 279, 373, 646, 214, 227, 244, 213, 268, 269, 349, 434, 435, 436, + 1006, 1015, 1017, 1263, 2671, 3417, 5008, 1424, 1151, 1792, 3876, 361, 527, 5869, 2184, 8402, 1482, 3222} + + +def norm(s): + s = unicodedata.normalize("NFKD", str(s)).encode("ascii", "ignore").decode("ascii") + return " ".join(re.sub(r"[^a-z0-9 ]+", " ", s.lower()).split()) + + +def stem(w): + w = w.lower() + if w.endswith("ied") and len(w) <= 5: # died -> die, lied -> lie + return w[:-1] + for suf in ("ations", "ation", "ings", "ing", "ers", "ors", "er", "or", "ed", "es", "s", "ion"): + if len(w) > len(suf) + 3 and w.endswith(suf): + return w[: -len(suf)] + return w + + +_lab, _cls = {}, {} + + +def label(q): + if q not in _lab: + r = db.execute("SELECT label, enwiki FROM entity WHERE qid=?", (q,)).fetchone() + _lab[q] = (r[0] or r[1]) if r else None + return _lab[q] + + +def classes(q): + if q in _cls: + return _cls[q] + out, frontier, seen = set(), [o for (o,) in db.execute("SELECT o FROM fact WHERE s=? AND p=31", (q,)) if o], set() + for d in range(2): # its class and one parent: higher superclasses name things it is not + nxt = [] + for c in frontier: + if c in seen: + continue + seen.add(c) + lab = label(c) + if lab: + ws = norm(lab).split() + if ws: + out.add(ws[-1]) # head noun: "town in Finland" is a town, not a Finland + if d < 1: + nxt += [o for (o,) in db.execute("SELECT o FROM fact WHERE s=? AND p=279", (c,)) if o] + frontier = nxt + _cls[q] = out + return out + + +_cls0 = {} +GEO_SYN = {"country": "country state kingdom republic nation monarchy federation empire principality sultanate emirate", + "city": "city town capital metropolis municipality borough commune", "town": "town city municipality borough commune village", + "state": "state province", "county": "county shire", "island": "island atoll", "river": "river stream tributary", + "mountain": "mountain peak volcano massif summit", "lake": "lake reservoir", "sea": "sea", "ocean": "ocean"} +GEO_SYN = {stem(k): set(stem(x) for x in v.split()) for k, v in GEO_SYN.items()} + + +def classes0(q): + """head nouns of the thing's own classes (P31 only): a city is a city, not the country its classes sit under""" + if q not in _cls0: + out = set() + for (o,) in db.execute("SELECT o FROM fact WHERE s=? AND p=31", (q,)): + lab = label(o) if o else None + ws = norm(lab).split() if lab else [] + if ws: + out.add(stem(ws[-1])) + _cls0[q] = out + return _cls0[q] + + +def is_human(q): + return db.execute("SELECT 1 FROM fact WHERE s=? AND p=31 AND o=5 LIMIT 1", (q,)).fetchone() is not None + + +def ent(title): + t = title.strip(" .,'\"") + if not t: + return None + for v in dict.fromkeys((t, t[:1].upper() + t[1:], " ".join(w[:1].upper() + w[1:] for w in t.split()))): + r = db.execute("SELECT qid FROM entity WHERE enwiki=? LIMIT 1", (v,)).fetchone() + if r: + return r[0] + return None + + +def linked(q): + words = re.findall(r"[A-Za-z0-9'\-\.&]+", q.replace('"', " ")) + taken, out = set(), [] + for n in (6, 5, 4, 3, 2, 1): + for i in range(len(words) - n + 1): + if any(j in taken for j in range(i, i + n)): + continue + span = words[i:i + n] + if span[0].lower() in STOP or span[-1].lower() in STOP: + continue + if n == 1 and (not span[0][:1].isupper() or i == 0 or len(span[0]) < 3): + continue + if all(w.lower() in STOP for w in span): + continue + before = words[i - 1] if i > 0 else "" + after = words[i + n] if i + n < len(words) else "" + if (after[:1].isupper() and after.lower() not in STOP) or (i > 1 and before[:1].isupper() and before.lower() not in STOP): + continue # part of a longer name ("Andrew Bonar" in "Andrew Bonar Law") + qid = ent(" ".join(span)) + if qid: + out.append((" ".join(span), qid)) + taken.update(range(i, i + n)) + return out + + +def asked_type(q): + ql = q.lower().strip() + w = re.findall(r"[a-z]+", ql) + if not w: + return None, set() + if w[0] in HUMAN_Q or " who " in " " + ql[:40] or re.search(r"\bto whom\b|\bwhom\b", ql): + return "human", set() + m = re.search(r"\b(?:which|what)\s+((?:[a-z\-]+\s+){0,2}?[a-z\-]+)\s*\?\s*$", ql) # "... the sister of which actor?" + if m: + nouns = set(stem(x) for x in m.group(1).split() if x not in STOP and x not in GENERIC_TYPE) + if nouns: + return "noun", nouns + m = re.match(r"(?:in )?(?:which|what) (city|town|country|state|county|island|river|mountain|lake|sea|ocean)\b", ql) + if m: + return "noun", {stem(m.group(1))} + if w[0] == "where": + return "place", set() + if w[0] == "when" or re.match(r"(in )?(what|which) year", ql): + return "time", set() + if re.match(r"how (many|much|long|old|far|tall|high)", ql): + return "number", set() + m = re.match(r"(?:what|which|who) (?:is|was|are|were) the ([a-z\-]+(?: [a-z\-]+)?) of\b", ql) # "what is the capital of ..." + if m: + head = m.group(1).split()[-1] + if head not in STOP and head not in GENERIC_TYPE: + return "rel", {stem(head)} + m = re.search(r"\b(which|what)\s+((?:[a-z\-]+\s+){0,3}?)([a-z\-]+)\s+(?:" + "|".join(AUX) + r"|[a-z]+ed|[a-z]+s)\b", ql) + if m: + nouns = set(stem(x) for x in (m.group(2) + " " + m.group(3)).split() if x not in STOP and x not in GENERIC_TYPE) + if nouns: + return "noun", nouns + return None, set() + + +_pw = {} + + +def prop_words(p): + if p not in _pw: + s = set() + for name in PROPNAMES.get(p, [PROP.get(p, "")]): + ws = [stem(x) for x in norm(name).split() if x not in STOP and len(x) > 2] + if 0 < len(ws) <= 3: # long descriptive aliases are not cues + s.update(ws) + _pw[p] = s + return _pw[p] + + +# words too loose to name a relation on their own (dev 10-14): "wrote" fits author, lyricist and publisher; "song" fits lyricist; +# "found" (in "found in the museum") stems like "founded" +STOPCUE = {stem(x) for x in ("write wrote written writ song songs theme found find make made play played sing sang known book books " + "music musical film films movie movies national").split()} + + +def cued(p, qwords_stem, qwords): + qwords_stem = qwords_stem - STOPCUE + if prop_words(p) & qwords_stem: + return True + if p not in ALIAS_STEMS: + ALIAS_STEMS[p] = set(stem(x) for x in ALIASES.get(p, "").split()) + if ALIAS_STEMS[p] & qwords_stem: + return True + return False # cue pairs dropped: "which|city" is not a relation (dev 10-14) + + +FEMININE = {"actress": "actor", "authoress": "author", "poetess": "poet", "comedienne": "comedian", "sculptress": "sculptor", + "heroine": "hero", "princess": "prince", "duchess": "duke", "empress": "emperor", "songstress": "singer"} + + +def type_ok(kind, nouns, cand_qid, lit, p=None): + if kind == "rel": + # the relation noun itself ("capital") must be the property: named by the property's own label or aliases + return p is not None and bool(prop_words(p) & nouns) and (cand_qid is not None or lit is not None) + if kind == "time": + return lit is not None and bool(re.search(r"\b1?\d{3,4}\b", str(lit))) + if cand_qid is None: + return False + if kind == "human": + return is_human(cand_qid) + if kind == "place": + return bool(classes(cand_qid) & PLACE_WORDS) + if kind == "noun": + nouns = set(stem(FEMININE.get(x, x)) for x in nouns) | nouns + cw = set(stem(x) for x in classes(cand_qid)) + if "person" in nouns or "man" in nouns or "woman" in nouns: + return is_human(cand_qid) + geo = [n for n in nouns if n in GEO_SYN] + if geo: # "which country": Nairobi is a city (held-out 10-14: Peter Hain) + return any(classes0(cand_qid) & GEO_SYN[n] for n in geo) + return bool(cw & nouns) or (is_human(cand_qid) and bool(nouns & occupations(cand_qid))) + return False + + +_occ = {} + + +def occupations(q): + if q not in _occ: + s = set() + for (o,) in db.execute("SELECT o FROM fact WHERE s=? AND p IN (106, 39) LIMIT 30", (q,)): + if o and label(o): + ws = norm(label(o)).split() + if ws: + s.add(stem(ws[-1])) # the head noun only: "film score composer" is a composer + _occ[q] = s + return _occ[q] + + +_gen = {} + + +def generic(e): + """a class or concept, not a particular thing: it is a subclass of something, or many things are instances of it""" + if e not in _gen: + sub = db.execute("SELECT 1 FROM fact WHERE s=? AND p=279 LIMIT 1", (e,)).fetchone() is not None + inst = len(db.execute("SELECT s FROM fact WHERE p=31 AND o=? LIMIT 30", (e,)).fetchall()) >= 30 + occ = len(db.execute("SELECT s FROM fact WHERE p=106 AND o=? LIMIT 30", (e,)).fetchall()) >= 30 + _gen[e] = sub or inst or occ + return _gen[e] + + +_word_props = None + + +def inv_props(qwords_stem, qwords): + """properties whose label/alias words or cue pairs appear in the question (index built once; was a scan of 38k per entity)""" + global _word_props + if _word_props is None: + _word_props = {} + for p in set(PROPNAMES) | set(PROP): + if p in SKIP_PROPS: + continue + for w in prop_words(p) | set(stem(x) for x in ALIASES.get(p, "").split()): + _word_props.setdefault(w, set()).add(p) + ps = set() + for w in qwords_stem - STOPCUE: + ps |= _word_props.get(w, set()) + ps = [p for p in ps if db.execute("SELECT 1 FROM fact WHERE p=? LIMIT 1", (p,)).fetchone()] + return ps[:60] + + +SHORT_NAMES = {"uk": 145, "u k": 145, "britain": 145, "great britain": 145, "us": 30, "u s": 30, "usa": 30, "u s a": 30, + "america": 30, "ussr": 15180, "soviet union": 15180, "uae": 878, "eu": 458, "holland": 55} + + +def echoed(q, val, ans): + """the answer is already in the question: by its label, or by a common short name (dev-burned held-out 10-14: "Where + are Akrotiri and Dhekelia, allocated by treaty to the UK" read the holder, the UK, as the place)""" + qn = " %s " % norm(q) + words = [w for w in norm(ans).split() if w not in STOP and w not in ("the", "of")] + if words and all(" %s " % w in qn for w in words): + return True + return val is not None and any(" %s " % k in qn and v == val for k, v in SHORT_NAMES.items()) + + +QWORDS = set("who whom whose what which where when why how in on at of the a an is was were are did does do name".split()) + + +def unlinked_name(q, links): + """a capitalised name or a quoted title in the question that no linked span covers""" + covered = set() + for t, _ in links: + covered.update(w.lower() for w in re.findall(r"[A-Za-z0-9']+", t)) + for quoted in re.findall(r"[\"“]([^\"”]{2,80})[\"”]", q): + if not any(quoted.lower() in t.lower() or t.lower() in quoted.lower() for t, _ in links): + return True + words = re.findall(r"[A-Za-z][A-Za-z0-9'\-]*", q) + for i, w in enumerate(words): + if i == 0 or not w[:1].isupper() or w.lower() in QWORDS or w.lower() in STOP: + continue + if w.lower() not in covered: + return True + return False + + +def answer(q): + if QUALIFIER.search(q): + return None + kind, nouns = asked_type(q) + if kind in (None, "number"): + return None + ents = [(t, e) for t, e in linked(q) if not generic(e)] + if not ents: + return None + if unlinked_name(q, linked(q)): + return None # held-out 10-14: Jennings Brewery -> read Cumbria; Dismaland -> Banksy + if q.count(",") >= 2 or (re.search(r"\band\b", q) and len(ents) >= 2): + return None # several clues joined: one fact cannot answer it + if kind == "rel": + m = re.search(r"\bof (?:the )?(.+?)(?:\?|$| in | on | at )", q) + if not m: + return None + first = [(t, e) for t, e in ents if m.group(1).lower().startswith(t.lower())] + if not first: + return None + ents = first[:1] # "the capital of the Yukon Territory in Canada": the Yukon, not Canada + named = {qid for _, qid in ents} + qwords = set(re.findall(r"[a-z]+", q.lower())) + qwords_stem = set(stem(x) for x in qwords if x not in STOP) + if kind == "noun": + qwords_stem -= nouns # the answer-type word is not the relation ("which country ... headquarters") + cands = {} + for span, e in ents: + for p, o, lit in db.execute("SELECT p, o, lit FROM fact WHERE s=? LIMIT 4000", (e,)): + if p in SKIP_PROPS or p not in ALLOW or (o is not None and o in named): + continue + if o is not None and (label(o) or "").startswith("Category:"): + continue + if not cued(p, qwords_stem, qwords): + continue + if type_ok(kind, nouns, o, lit, p): + key = ("q", o) if o is not None else ("l", str(lit)) + cands.setdefault(key, []).append((e, p, "fwd")) + for p in [x for x in inv_props(qwords_stem, qwords) if x in ALLOW]: + # reverse answers only when the relation's MAIN name is in the question: aliases such as "mother of" on the + # child relation describe the relation backwards (dev 10-14: "mother of Achilles") + main = set(stem(x) for x in norm(PROP.get(p, "")).split() if x not in STOP and len(x) > 2) + if not (main & qwords_stem): + continue + subs = [s for (s,) in db.execute("SELECT s FROM fact WHERE p=? AND o=? LIMIT 6", (p, e))] + if len(subs) > 5: # a big group (everyone who received an award): no reverse answer + continue + for s in subs: + if s in named: + continue + if type_ok(kind, nouns, s, None, p): + cands.setdefault(("q", s), []).append((e, p, "inv")) + if len(cands) != 1: + return None + (kt, val), why = next(iter(cands.items())) + if len(ents) >= 2 and len({w[0] for w in why}) < 2: # two or more named things: at least two must agree (held-out + return None # 10-14: Jennings Brewery -> Cumbria's capital, Dismaland -> Banksy) + if False: + return None + e, p, d = why[0] + ans = label(val) if kt == "q" else val + if not ans: + return None + if echoed(q, val if kt == "q" else None, ans): + return None # the question already names it ("allocated to the UK": not where it is) + if d == "fwd": + receipt = "%s (Q%d) - %s (P%d) - %s" % (label(e), e, PROP.get(p, ""), p, ("%s (Q%d)" % (ans, val)) if kt == "q" else ans) + else: + receipt = "%s (Q%d) - %s (P%d) - %s (Q%d)" % (ans, val, PROP.get(p, ""), p, label(e), e) + return {"answer": ans, "receipt": receipt, "prop": p, "dir": d, "kind": kind, "agree": len(why)} + + +def correct(ans, golds): + """exact match after normalising, or one contains the other as whole words ("Lake Mead" / "Mead", "The Dirham" / + "United Arab Emirates dirham"); articles dropped""" + drop = lambda s: " ".join(w for w in norm(s).split() if w not in ("the", "a", "an")) + a = drop(ans) + for g in (drop(x) for x in golds if x): + if not g: + continue + if a == g: + return True + if len(g) >= 4 and re.search(r"\b%s\b" % re.escape(g), a): + return True + if len(a) >= 4 and re.search(r"\b%s\b" % re.escape(a), g): + return True + return False + + +if __name__ == "__main__": + if sys.argv[1] == "ask": + print(json.dumps(answer(sys.argv[2]), indent=1)) + sys.exit() + import multiprocessing as mp + + def one(item): + q, golds = item + try: + r = answer(q) + except Exception as e: # noqa: BLE001 + return ("err", q[:80], str(e)[:80]) + if not r: + return None + return ("right" if correct(r["answer"], golds) else "wrong", q[:110], r["answer"], golds[:2], r["receipt"][:130], r["kind"]) + + items = [it for f in sys.argv[2:] for it in json.load(open(f))] + cnt = {"right": 0, "wrong": 0, "err": 0} + rows = [] + with mp.Pool(6) as pool: + for i, res in enumerate(pool.imap_unordered(one, items, chunksize=4), 1): + if res: + cnt[res[0]] += 1 + rows.append(res) + if i % 250 == 0: + print("progress %d/%d right %d wrong %d" % (i, len(items), cnt["right"], cnt["wrong"]), flush=True) + fired = cnt["right"] + cnt["wrong"] + print("questions %d fired %d right %d wrong %d errors %d precision %.3f" % (len(items), fired, cnt["right"], cnt["wrong"], cnt["err"], cnt["right"] / max(1, fired))) + for r in rows: + if r[0] != "right": + print(" ", r[0].upper(), r[1:]) + json.dump(rows, open("/root/w240/coverage_eval_last.json", "w"), indent=0) diff --git a/root/wander-train/math_organ_v2.py b/root/wander-train/math_organ_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..cbb5bcbb8c98feba202202281cc179c17aea2f0d --- /dev/null +++ b/root/wander-train/math_organ_v2.py @@ -0,0 +1,508 @@ +"""math_organ_v2 - v1 (admitted in w480, now immutable) + two fixes found on the gate panel: "factorial" is not a +request to factor, and a request for a PROGRAM is the code organ's, not a mathematical request. + +exact mathematics as specified free moves, every result independently checked, the method cited. + +REQUEST directive (solve / simplify / factor / expand / evaluate / differentiate / integrate / limit / decompose) + + the expression(s), read from the question or from its exercise set's instruction +MOVES SymPy operations legal for that directive and that expression's type +CHECK independent of the move that produced the answer: + solve every solution substituted back satisfies every equation / inequality + factor the product expands back to the input, and is a product + simplify equivalent to the input at random points, and no more complex than the input + evaluate recomputed by direct numeric substitution + derivative agrees with a central difference at random points + integral differentiates back to the integrand (definite: numeric quadrature agrees) + limit agrees with evaluation near the point +STORED FORM the textbook's own method for the directive (a howto or a worked example, textbook_store_v1) is cited +Nothing is asserted unless the check passes; a failed check is an abstention with the reason.""" +from __future__ import annotations + +import random +import re +import sqlite3 + +import sympy as sp +from sympy.parsing.sympy_parser import (convert_xor, implicit_multiplication_application, parse_expr, + split_symbols_custom, standard_transformations) + +STORE = __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/textbooks/textbook_store_v1.sqlite3" +FUNCS = {"sin", "cos", "tan", "sec", "csc", "cot", "log", "ln", "exp", "sqrt", "root", "asin", "acos", "atan", "abs", + "sinh", "cosh", "tanh", "pi", "oo", "theta", "alpha", "beta", "gamma", "Delta", "lambda", "mu", "sigma", + "omega", "phi", "rho", "tau", "epsilon", "arcsin", "arccos", "arctan"} +LOCAL = {"e": sp.E, "pi": sp.pi, "ln": sp.log, "log": sp.log, "root": lambda x, n: sp.root(x, n), "oo": sp.oo, + "arcsin": sp.asin, "arccos": sp.acos, "arctan": sp.atan, "abs": sp.Abs, "theta": sp.Symbol("theta")} + + +def _split_ok(name): + return name not in FUNCS and not name.startswith(("theta", "alpha", "beta", "gamma", "Delta", "lambda", "omega")) + + +TRANSF = standard_transformations + (split_symbols_custom(_split_ok), implicit_multiplication_application, convert_xor) +DIRECTIVES = [ + ("partial", r"partial fraction|decomposition"), + ("derivative", r"\bderivative|differentiate|\bf'\(|d/dx|dy/dx"), + ("integral", r"\bintegra(l|te)|antiderivative"), + ("limit", r"\blimit|\blim_"), + ("factor", r"\bfactor(?!ial)"), # v2: "factorial" is not a request to factor + ("solve", r"\bsolve|\bsolution|find (?:the )?(?:value|zeros?|roots?)\b|\bfind x\b"), + ("evaluate", r"\bevaluate|\bfind f\(|\bcompute|\bcalculate|\bwhat is\b|\bhow much is\b"), + ("expand", r"\bmultiply|\bexpand|\bdistributive"), + ("simplify", r"\bsimplif|\badd\b|\bsubtract|\bdivide|\bcombine|\breduce"), +] + + +def clean_linear(s): + s = s.replace("−", "-").replace("×", "*").replace("·", "*").replace("÷", "/").replace("^", "**") + s = re.sub(r"(\d),(\d{3})\b", r"\1\2", s) + s = re.sub(r"\s+", " ", s).strip().rstrip(".?,;") + return s + + +def P(s, syms=None): + s = clean_linear(s) + s = re.sub(r"\bsqrt\s*\(", "sqrt(", s) + return parse_expr(s, local_dict=dict(LOCAL, **(syms or {})), transformations=TRANSF, evaluate=True) + + +def directive_of(text): + t = (text or "").lower() + for name, rx in DIRECTIVES: + if re.search(rx, t): + return name + return None + + +ENGLISH = {"of", "to", "is", "by", "at", "in", "on", "if", "an", "as", "the", "and", "for", "use", "let", "all", "are", + "its", "how", "not", "or", "be", "it", "we", "find", "when", "each", "given", "then", "from", "with", "that", + "this", "what", "which", "where", "your", "you", "into", "using", "value", "values", "following", "exercises", + "expression", "expressions", "equation", "equations", "simplify", "solve", "factor", "evaluate", "multiply", + "divide", "add", "subtract", "completely", "polynomial", "form", "write", "answer", "result", "greatest", + "common", "number", "numbers", "term", "terms", "exact", "calculator", "round", "rule", "any", "one", "two", + "has", "can", "our", "was", "per", "sum", "set", "get", "may", "see", "new", "own", "way", "out"} +TOK = re.compile(r"[A-Za-z]+|\d+(?:\.\d+)?|\*\*|<=|>=|!=|->|\+-|[-+*/^=<>(){}\[\],|_;'!]|\.(?=\d)|\.|\s+") + + +def _is_math_word(w, prev, nxt): + lw = w.lower() + if lw in FUNCS or w == "C": + return True + if lw in ENGLISH: + return False + if len(w) == 1: + return not (w in ("a", "A", "I") and nxt.isspace() and prev in ("", " ")) + return len(w) <= 3 # implicit products "xy", "ah", "abc" + + +FUNC_RX = r"(sqrt|sin|cos|tan|sec|csc|cot|log|ln|exp)" + + +def normalize_request(text): + """surface conventions -> linear mathematics: '2xsqrt(48)' -> '2x sqrt(48)', degrees, log bases, imaginary unit""" + t = text or "" + t = re.sub(r"(?<=[a-z0-9)])%s(?=[\s(])" % FUNC_RX, r" \1", t) # xsqrt( -> x sqrt( + t = re.sub(r"(\d+(?:\.\d+)?)\s*°", r"(\1*pi/180)", t) # 120° -> radians + t = re.sub(r"\blog\s*_\s*\{?(\d+)\}?\s*\(?([^()\s]+)\)?", r"log(\2,\1)", t) # log_5 25 -> log(25,5) + if re.search(r"\bcomplex|\bimaginary|\d\s*i\b|[-+]\s*\d*\s*i\)", t, re.I): + t = re.sub(r"(?()]", s)) and re.search(r"[a-z0-9]", s): + out.append(s) + return out + + +def assignments(text): + """'when a=-4, b=6' / 'for x = 2 and y = 3' -> {a: -4, b: 6}""" + out = {} + m = re.search(r"\b(?:when|for|if|with|where|at)\s+((?:[a-z]\s*=\s*[-+]?[0-9./()]+\s*(?:,|\band\b)?\s*)+)", text or "") + if m: + for v, val in re.findall(r"\b([a-z])\s*=\s*([-+]?[0-9./()]+)", m.group(1)): + try: + out[sp.Symbol(v)] = P(val) + except Exception: # noqa: BLE001 + pass + return out + + +def target_var(text): + m = re.search(r"\bfor\s+([a-z])\b(?!\s*=)", text or "") + return sp.Symbol(m.group(1)) if m else None + + +def rounding(text): + t = (text or "").lower() + m = re.search(r"round(?:ed)?\s+(?:your answer\s+)?to\s+(?:the\s+nearest\s+)?(\w+)", t) + if not m: + return None + return {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "tenth": 1, "hundredth": 2, "thousandth": 3, + "whole": 0, "integer": 0, "1": 1, "2": 2, "3": 3, "4": 4}.get(m.group(1)) + + +def binary_phrase(text): + """'the difference of A and B' -> (A)-(B); sum / product / quotient likewise; 'subtract A from B' -> (B)-(A)""" + m = re.search(r"\bsubtract\s+(.+?)\s+from\s+(.+?)(?:\.\s|\.$|$)", text or "", re.I) + if m: + a, b = extract_math(m.group(1)), extract_math(m.group(2)) + return "(%s)-(%s)" % (b[0], a[-1]) if a and b else None + m = re.search(r"\b(sum|difference|product|quotient) of (.+?) and (.+?)(?:\.\s|\.$|$)", text or "", re.I) + if not m: + return None + op = {"sum": "+", "difference": "-", "product": "*", "quotient": "/"}[m.group(1).lower()] + a, b = extract_math(m.group(2)), extract_math(m.group(3)) + return "(%s)%s(%s)" % (a[-1], op, b[0]) if a and b else None + + +def complexity(e): + return sp.count_ops(e, visual=False) + len(str(e)) / 40.0 + + +def rand_pts(syms, k=5): + rng = random.Random(7) + return [{s: sp.Rational(rng.randint(2, 29), rng.randint(3, 11)) for s in syms} for _ in range(k)] + + +def equiv(a, b): + try: + d = sp.simplify(a - b) + if d == 0: + return True + syms = sorted((a.free_symbols | b.free_symbols), key=str) + for pt in rand_pts(syms): + va, vb = complex(a.subs(pt).evalf()), complex(b.subs(pt).evalf()) + if abs(va - vb) > 1e-7 * max(1.0, abs(va)): + return False + return True + except Exception: # noqa: BLE001 + return False + + +class MathOrgan: + def __init__(self, store=STORE): + try: + self.db = sqlite3.connect("file:%s?mode=ro" % store, uri=True, check_same_thread=False) + except Exception: # noqa: BLE001 + self.db = None + self.why = "" + + # ------------------------------------------------------------------ the stored form (method citation) + def method(self, directive, hint=""): + if not self.db: + return None + words = {"solve": "solve equation", "simplify": "simplify expression", "factor": "factor", "expand": "multiply", + "evaluate": "evaluate", "derivative": "derivative", "integral": "integral", "limit": "limit", + "partial": "partial fraction decomposition"}[directive] + q = " OR ".join(w for w in words.split()) + try: + r = self.db.execute("SELECT item.book, item.section, item.title, item.kind, item.id FROM item_fts JOIN item " + "ON item.id = item_fts.rowid WHERE item_fts MATCH ? AND item.kind IN ('howto','example') " + "ORDER BY (item.kind='howto') DESC, rank LIMIT 1", ("title:(%s)" % q,)).fetchone() + except Exception: # noqa: BLE001 + r = None + return {"book": r[0], "section": r[1], "title": r[2], "kind": r[3], "id": "tb:%d" % r[4]} if r else None + + # ------------------------------------------------------------------ the solve + def answer(self, question, instruction=None): + self.why = "" + # v2: a request for a PROGRAM ("write a Python function that returns the factorial ...") is the code organ's + if re.search(r"\b(?:write|make|create|implement|code)\b.*\b(?:function|program|script|method)\b", question or "", re.I): + self.why = "a request for a program, not for a mathematical result" + return None + # v1 reads LINEAR notation (the textbook store's form, and what people type). Typeset LaTeX ($...$, \frac, + # \sqrt{}) is declined until a LaTeX reader exists: reading it with the linear reader would be luck, not reading. + if re.search(r"\$|\\(?:frac|sqrt|cdot|left|right|begin|dfrac|tfrac|pi|times)\b", question or ""): + self.why = "typeset (LaTeX) mathematics: this organ reads linear notation only" + return None + question = normalize_request(question) + d = directive_of(question) or directive_of(instruction or "") + # a WORD PROBLEM needs language -> equations (a later organ); reading numbers out of prose is guessing + body = question[len(instruction):] if instruction and question.startswith(instruction) else question + if english_load(body) > WORD_PROBLEM_WORDS: + self.why = "a word problem: the quantities must be translated into equations first" + return None + bp = binary_phrase(question) + spans = [bp] if bp else extract_math(question) + # a request whose only mathematics is a bare number names an operation this organ does not hold + # ("the lcm of 73 and 230", "452 in base 7", "803 in roman numerals"): echoing the number would be a guess + if not any(re.search(r"[-+*/^=<>()]|[a-z]", s_) for s_ in spans): + self.why = "no expression to operate on: the request names an operation this organ does not hold" + return None + if re.search(r"\b(?:lcm|gcd|gcf|base\s+\d|roman|numerals?|binary|hexadecimal|octal|convert|km|miles?|yards?|feet|" + r"inch(?:es)?|meters?|kilograms?|pounds?|percent|%)\b", question, re.I) and d in ("evaluate", "simplify"): + self.why = "a named operation or unit this organ does not hold" + return None + defined = {m.group(1) for s_ in spans for m in [re.match(r"^\s*([a-zA-Z])\s*\(\s*[a-z]\s*\)\s*=", s_)] if m} + used = {m.group(1) for s_ in spans for m in re.finditer(r"(?" in s for s in spans) else "simplify" + try: + res = getattr(self, "_" + d)(spans, question) + except Exception as e: # noqa: BLE001 + self.why = "%s failed: %s" % (d, repr(e)[:80]) + return None + if not res: + self.why = self.why or "%s: no checked result" % d + return None + nd = rounding(question) + if nd is not None and isinstance(res.get("value"), sp.Basic) and res["value"].is_number: + v = round(float(res["value"].evalf()), nd) + res["value"], res["answer"] = sp.Float(v), ("%d" % v if nd == 0 else ("%." + str(nd) + "f") % v) + res["steps"] = res.get("steps", []) + ["round to %d decimal place%s" % (nd, "" if nd == 1 else "s")] + res["directive"] = d + res["method"] = self.method(d) + return res + + # --- solve: equations, inequalities, systems ------------------------------------------------------------------ + def _solve(self, spans, q): + eqs = [] + for s in spans: + for part in re.split(r";|\band\b", s): + part = part.strip() + m = re.match(r"^(.*?)(<=|>=|<|>|=)(.*)$", part) + if not m: + continue + lhs, op, rhs = m.groups() + if not lhs.strip() or not rhs.strip(): + continue + L_, R_ = P(lhs), P(rhs) + eqs.append((L_, op, R_)) + if not eqs: + return None + syms = sorted(set().union(*[(a.free_symbols | b.free_symbols) for a, _, b in eqs]), key=str) + tv = target_var(q) + if tv is not None and tv in syms: + syms = [tv] + elif len(eqs) < len(syms): + self.why = "more unknowns than equations and no variable named" + return None + if not syms or len(syms) > 3: + return None + if all(op == "=" for _, op, _ in eqs): + sols = sp.solve([sp.Eq(a, b) for a, _, b in eqs], syms, dict=True) + if not sols: + self.why = "no solution found" + return None + for sol in sols: # CHECK: substitute back + for a, _, b in eqs: + if not equiv(a.subs(sol), b.subs(sol)): + self.why = "substitution check failed" + return None + out = [", ".join("%s = %s" % (k, sp.sstr(v)) for k, v in sorted(sol.items(), key=lambda kv: str(kv[0]))) + for sol in sols] + return {"answer": " or ".join(out), "value": sols, "check": "each solution substituted back satisfies every equation", + "steps": ["read %s" % "; ".join("%s %s %s" % (sp.sstr(a), o, sp.sstr(b)) for a, o, b in eqs), + "solve for %s" % ", ".join(map(str, syms))]} + if len(eqs) == 1 and len(syms) == 1: + a, op, b = eqs[0] + rel = {"<": sp.Lt, ">": sp.Gt, "<=": sp.Le, ">=": sp.Ge}[op](a, b) + sol = sp.solveset(rel, syms[0], domain=sp.S.Reals) + # CHECK: a point inside satisfies, a point outside does not + inside = sol.inf + 1 if sol.inf.is_finite else (sol.sup - 1 if sol.sup.is_finite else 0) + if inside not in sol or not rel.subs(syms[0], inside): + self.why = "inequality check failed" + return None + return {"answer": sp.sstr(sol), "value": sol, "check": "a point of the solution set satisfies the inequality", + "steps": ["read %s %s %s" % (sp.sstr(a), op, sp.sstr(b)), "solve over the reals"]} + return None + + # --- factor --------------------------------------------------------------------------------------------------- + def _factor(self, spans, q): + e = P(max(spans, key=len)) + f = sp.factor(e) + if not equiv(sp.expand(f), sp.expand(e)): + return None + if not (isinstance(f, sp.Mul) or isinstance(f, sp.Pow)) and f == e: + self.why = "does not factor over the rationals" + return None + return {"answer": sp.sstr(f), "value": f, "check": "the product expands back to the input", + "steps": ["read %s" % sp.sstr(e), "factor completely"]} + + # --- simplify / expand ---------------------------------------------------------------------------------------- + def _simplify(self, spans, q, how="simplify"): + e0 = P(max(spans, key=len)) + # the textbook convention: variables under radicals and exponents are taken as positive + pos = {s: sp.Symbol(str(s), positive=True) for s in e0.free_symbols} + e = e0.xreplace(pos) + s0 = sp.simplify(e) + cands = [s0, sp.together(s0), sp.powsimp(s0), sp.powdenest(s0, force=True), sp.expand(e), sp.cancel(e), + sp.factor(e), sp.radsimp(s0)] + back = {v: k for k, v in pos.items()} + cands = [c.xreplace(back) for c in cands] + e = e0 + if how == "expand": + cands = [sp.expand(e)] + cands + best = cands[0] + else: + best = min(cands, key=complexity) + if not equiv(best, e): + return None + return {"answer": sp.sstr(best), "value": best, + "check": "equivalent to the input at random points (variables positive, the textbook convention)", + "steps": ["read %s" % sp.sstr(e), "%s" % ("expand" if how == "expand" else "simplify")]} + + def _expand(self, spans, q): + return self._simplify(spans, q, how="expand") + + # --- evaluate ------------------------------------------------------------------------------------------------- + def _evaluate(self, spans, q): + fdef = next((s for s in spans if re.match(r"^\s*[a-zA-Z]\s*\(\s*[a-z]\s*\)\s*=", s)), None) + if fdef: + m = re.match(r"^\s*([a-zA-Z])\s*\(\s*([a-z])\s*\)\s*=(.*)$", fdef) + fname, var, body = m.groups() + f = P(body) + x = sp.Symbol(var) + args = [re.match(r"^\s*%s\s*\((.*)\)\s*$" % fname, s) for s in spans if s is not fdef] + args = [a.group(1) for a in args if a] + if not args: + return None + outs = [] + for a in args: + v = sp.simplify(f.subs(x, P(a))) + chk = f.subs(x, P(a)) + if not equiv(v, chk): + return None + outs.append("%s(%s) = %s" % (fname, a, sp.sstr(v))) + return {"answer": "; ".join(outs), "value": outs, "check": "recomputed by direct substitution", + "steps": ["read %s(%s) = %s" % (fname, var, sp.sstr(f))] + ["substitute %s" % a for a in args]} + subs = assignments(q) + cand = [s_ for s_ in spans if not re.match(r"^\s*[a-z]\s*=\s*[-+]?[0-9./()]+\s*$", s_)] or spans + e = P(max(cand, key=len).split("=")[0]) + if subs and e.free_symbols and e.free_symbols <= set(subs): + v = sp.nsimplify(sp.simplify(e.subs(subs))) + if not equiv(v, e.subs(subs)): + return None + return {"answer": sp.sstr(v), "value": v, "check": "recomputed by direct substitution", + "steps": ["read %s" % sp.sstr(e), "substitute %s" % ", ".join("%s = %s" % kv for kv in subs.items())]} + if e.free_symbols: + return self._simplify(spans, q) + v = sp.nsimplify(sp.simplify(e)) if not e.is_Float else e + if abs(complex(v.evalf()) - complex(e.evalf())) > 1e-9 * max(1, abs(complex(e.evalf()))): + return None + return {"answer": sp.sstr(v), "value": v, "check": "recomputed numerically", + "steps": ["read %s" % sp.sstr(e), "evaluate"]} + + # --- calculus ------------------------------------------------------------------------------------------------- + def _var(self, e, q): + syms = sorted(e.free_symbols, key=str) + m = re.search(r"with respect to ([a-z])|d/d([a-z])", q) + if m: + return sp.Symbol(m.group(1) or m.group(2)) + for pref in ("x", "t", "u", "theta"): + for s in syms: + if str(s) == pref: + return s + return syms[0] if syms else sp.Symbol("x") + + def _body(self, spans): + s = max(spans, key=len) + s = re.sub(r"^\s*[a-zA-Z]\s*\(\s*[a-z]\s*\)\s*=", "", s) + s = re.sub(r"^\s*y\s*=", "", s) + return s + + def _derivative(self, spans, q): + e = P(self._body(spans)) + x = self._var(e, q) + d = sp.simplify(sp.diff(e, x)) + for pt in rand_pts([x], 3): # CHECK: central difference + h = sp.Rational(1, 10 ** 6) + num = (e.subs(x, pt[x] + h) - e.subs(x, pt[x] - h)) / (2 * h) + if abs(complex(num.evalf()) - complex(d.subs(pt).evalf())) > 1e-4 * max(1, abs(complex(d.subs(pt).evalf()))): + return None + return {"answer": sp.sstr(d), "value": d, "check": "agrees with a central difference at random points", + "steps": ["read %s" % sp.sstr(e), "differentiate with respect to %s" % x]} + + def _integral(self, spans, q): + s = max(spans, key=len) + m = re.search(r"integral\s*_\s*\{?([^}^]+?)\}?\s*\^\s*\{?([^}]+?)\}?\s*(\(.*|[^(].*?)\s*(?:d([a-z]))?\s*$", s) + fb = re.search(r"from\s+([-\w./()]+)\s+to\s+([-\w./()]+)", q) + lo = hi = None + if m: + lo, hi, body, v = m.groups() + e = P(body) + elif fb: + lo, hi = fb.groups() + body = max([x for x in spans if x.strip() not in (lo, hi)] or spans, key=len) + e = P(re.sub(r"\s*d[a-z]\s*$", "", self._body([body]).replace("integral", ""))) + else: + e = P(re.sub(r"\s*d([a-z])\s*$", "", self._body(spans).replace("integral", ""))) + x = self._var(e, q) + F = sp.integrate(e, x) + if F.has(sp.Integral): + return None + if not equiv(sp.diff(F, x), e): # CHECK: differentiates back + return None + if lo is not None: + val = sp.simplify(F.subs(x, P(hi)) - F.subs(x, P(lo))) + quad = sp.Integral(e, (x, P(lo), P(hi))).evalf() + if abs(complex(val.evalf()) - complex(quad)) > 1e-6 * max(1, abs(complex(quad))): + return None + return {"answer": sp.sstr(val), "value": val, "check": "antiderivative differentiates back; quadrature agrees", + "steps": ["read integral of %s from %s to %s" % (sp.sstr(e), lo, hi), "antiderivative %s" % sp.sstr(F)]} + return {"answer": sp.sstr(F) + " + C", "value": F, "check": "differentiates back to the integrand", + "steps": ["read %s" % sp.sstr(e), "integrate with respect to %s" % x]} + + def _limit(self, spans, q): + s = max(spans, key=len) + m = re.search(r"lim\s*_\s*\{?\s*([a-z])\s*->\s*([^}\s]+?)\s*\}?\s*(\S.*?)\s*(?:,.*)?$", s) or \ + re.search(r"as ([a-z]) (?:approaches|->) ([-\w.]+)", q) + if not m: + return None + if len(m.groups()) == 3: + v, pt, body = m.groups() + e = P(body) + else: + v, pt = m.groups() + e = P(self._body([x for x in spans if "lim" not in x] or spans)) + x, a = sp.Symbol(v), P(pt) + L_ = sp.limit(e, x, a) + if L_.is_finite: # CHECK: evaluation near the point + near = complex(e.subs(x, a + sp.Rational(1, 10 ** 7)).evalf()) + if abs(near - complex(L_.evalf())) > 1e-3 * max(1, abs(complex(L_.evalf()))): + return None + return {"answer": sp.sstr(L_), "value": L_, "check": "agrees with evaluation near the point", + "steps": ["read the limit of %s as %s -> %s" % (sp.sstr(e), v, pt)]} + + def _partial(self, spans, q): + e = P(max(spans, key=len)) + x = self._var(e, q) + r = sp.apart(e, x) + if not equiv(r, e): + return None + return {"answer": sp.sstr(r), "value": r, "check": "recombines to the input", + "steps": ["read %s" % sp.sstr(e), "decompose into partial fractions"]} diff --git a/root/wander-train/oracle_cube_reader_v1_3.py b/root/wander-train/oracle_cube_reader_v1_3.py new file mode 100644 index 0000000000000000000000000000000000000000..0b814634771657fdff8bef10d954753799d03555 --- /dev/null +++ b/root/wander-train/oracle_cube_reader_v1_3.py @@ -0,0 +1,447 @@ +"""oracle_cube_reader_v1 - the reasoning cube, Phase A: read -> frame -> link -> expand -> bind -> check -> verify. + +Three graphs share one middle pillar (request, binding, evidence, surface). This module walks the first six moves: + + Q Malkuth->Yesod READ the surface opens obligations + Q Yesod->Tiferet FRAME the answer slot: its TYPE (person, year, date, number, place, or an instance of the question's + head noun), the relation words, the constraints + O Tiferet->Chokmah LINK the subject, by the store's own title index (longest word n-grams), never by guessing + from capital letters + K Chokmah->Chesed EXPAND the subject's own page, sentence by sentence, each with its claim id + K Chesed->Tiferet BIND a candidate of the required type, drawn from a sentence that carries the relation + O Geburah->Tiferet CHECK INDEPENDENT of the proposing sentence: an entity candidate's OWN page must say it is the + asked-for kind; the question's constraints must be attested in the binding sentence or on + the candidate's page; a value candidate (year, date, number) must be the only value of its + type bound to a relation word in its sentence + Q Tiferet->Keter VERIFY every obligation discharged, one candidate standing, or nothing is said + +Laws obeyed (reach-is-not-answer, 2026-09-10): a candidate is proposed BECAUSE it could fill the open slot, carries a +typed binding, and is checked by something that did not propose it. No hypernym bridging. Abstention is free; a wrong +answer costs a right one. The store is ORACLE_DEEP_BUILD, read-only. +""" +from __future__ import annotations + +import os +import re +import sqlite3 +import unicodedata + +DEEP = os.environ.get("ORACLE_DEEP_BUILD") or ( + __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/fable-content/oracle-m-series-2026-08-15/" + "m13e8_dbpedia_enslot_openstax_full_fable_bench_a_v1/wikipedia_specialist.sqlite3") + +STOP = set("""a an the of and or for to in on at by with from into about as is are was were be been being it its this that +these those which what who whom whose when where why how did does do has have had not no but if than so such also can +will would could should may might must shall there their his her he she they them we you your our i me my one first +last name named called known""".split()) +MEASURES = {"capacity", "population", "height", "length", "area", "number", "size", "weight", "depth", "width", "age", + "distance", "speed", "elevation", "altitude", "attendance", "budget", "revenue"} +ANSWER_WORDS = {"day", "month", "year", "date", "time", "when", "first", "last", "name", "names", "called", "second", + "number", "many", "much", "exact", "full", "people", "person", "two", "three"} +FRAME_NOUNS = {"name", "type", "kind", "sort", "word", "term", "title", "one", "thing", "part", "member"} +ACTION_BREAK = re.compile(r"^(is|was|were|are|did|does|do|has|had|have|won|played|made|became|served|wrote|directed|" + r"founded|released|located|called|named|known|used|held|given|awarded|built|born|died|developed|" + r"invented|created|published|produced|starred|in|of|for|by|to|at|on|from|with|that|whose|who|" + r"which|as|after|before|during|and|or)$", re.I) +MONTHS = "january february march april may june july august september october november december".split() +MONTH_RE = "|".join(m.capitalize() for m in MONTHS) +YEAR = re.compile(r"\b(1[0-9]{3}|20[0-2][0-9])\b") +DATE = re.compile(r"\b(\d{1,2} (?:%s) \d{3,4}|(?:%s) \d{1,2}, \d{3,4})\b" % (MONTH_RE, MONTH_RE)) +MONTH_YEAR = re.compile(r"\b((?:%s) \d{4})\b" % MONTH_RE) +COMPARE = re.compile(r"\b(or|out of|between|both|either|neither)\b.*\?$|\b(more|most|less|fewer|older|younger|earlier|later|" + r"longer|shorter|larger|smaller|bigger|taller|first)\b.*\b(or|and)\b", re.I) +NUMBER = re.compile(r"\b\d{1,3}(?:,\d{3})+\b|\b\d+(?:\.\d+)?\b") +NUMWORDS = {w: i for i, w in enumerate("zero one two three four five six seven eight nine ten eleven twelve".split())} +CAPSPAN = re.compile(r"\b([A-Z][\w'\.\-&]*(?:\s+(?:of|the|de|von|van|da|del|la|le|and|&)?\s*[A-Z0-9][\w'\.\-&]*){0,5})") +TOK = re.compile(r"[A-Za-z0-9][A-Za-z0-9'\-\.&]*") +# v1.2: a PERSON is shown by a birth/life-span parenthetical or an occupation predicate, never by a pronoun anywhere on the +# page (v1.1 accepted "MA" - Massachusetts - as a person because the page said "her") +OCCUPATIONS = (r"actor|actress|singer|songwriter|musician|composer|writer|author|poet|novelist|journalist|politician|" + r"lawyer|judge|player|footballer|cricketer|athlete|coach|manager|painter|artist|sculptor|architect|" + r"scientist|physicist|chemist|biologist|mathematician|astronomer|engineer|inventor|economist|philosopher|" + r"historian|director|producer|businessman|businesswoman|entrepreneur|explorer|soldier|general|admiral|" + r"officer|king|queen|emperor|prince|princess|pope|bishop|priest|monk|saint|president|minister|senator|" + r"governor|diplomat|activist|comedian|dancer|model|designer|photographer|surgeon|physician|nurse|" + r"professor|teacher|scholar|academic|chef|pilot|astronaut|wrestler|boxer|racing driver|cyclist|swimmer|" + r"tennis player|golfer|chess player|rapper|drummer|guitarist|pianist|conductor|filmmaker|screenwriter|" + r"cartoonist|illustrator|critic|editor|publisher|broadcaster|presenter|television personality|nobleman|" + r"noblewoman|lord|lady|duke|earl|count|countess|baron|chief|leader|missionary|theologian|psychologist") +PERSON_LEAD = re.compile(r"\(\s*born\b|\(\s*(?:c\.\s*)?\d{1,2}? ?\w* ?\d{3,4}\s*[–-]|\(\s*fl\.|" + r"\b(?:is|was)\s+(?:a|an|the)\s+(?:[\w\-]+\s+){0,4}?(?:%s)s?\b" % OCCUPATIONS, re.I) +NOUNISH = re.compile(r"(?:ers?|ors?|ions?|ments?|ness|ities|ity|ists?|ship)$", re.I) +PLACE_LEAD = re.compile(r"\b(city|town|village|country|state|province|county|region|capital|municipality|island|" + r"district|commune|borough|suburb|territory|nation|located in|metropolis|kingdom|republic)\b", re.I) + + +def norm(s): + s = unicodedata.normalize("NFKD", str(s)).encode("ascii", "ignore").decode("ascii") + return " ".join(re.sub(r"[^a-z0-9 ]+", " ", s.lower()).split()) + + +IRREG = {"won": "win", "win": "win", "wins": "win", "wrote": "write", "written": "write", "writes": "write", + "write": "write", "began": "begin", "begun": "begin", "became": "become", "become": "become", "led": "lead", + "leads": "lead", "lead": "lead", "ran": "run", "run": "run", "sang": "sing", "sung": "sing", "sing": "sing", + "held": "hold", "hold": "hold", "made": "make", "make": "make", "took": "take", "taken": "take", "take": "take", + "gave": "give", "given": "give", "give": "give", "came": "come", "come": "come", "went": "go", "met": "meet", + "meet": "meet", "left": "leave", "leave": "leave", "built": "build", "build": "build", "sold": "sell", + "sell": "sell", "bought": "buy", "buy": "buy", "taught": "teach", "teach": "teach", "fought": "fight", + "fight": "fight", "drew": "draw", "drawn": "draw", "draw": "draw", "flew": "fly", "fly": "fly", "grew": "grow", + "grow": "grow", "spoke": "speak", "speak": "speak", "chose": "choose", "chosen": "choose", "choose": "choose", + "drove": "drive", "drive": "drive", "born": "bear", "birth": "bear", "died": "die", "dies": "die", "die": "die", + "death": "die", "beat": "beat", "beaten": "beat", "shot": "shoot", "hit": "hit", "set": "set", "put": "put"} +BASE_VERBS = set("""play join attend marry appear found discover invent release direct compose coach serve elect award +sign command host produce publish open close design create capture defeat star record graduate study move live name +score represent debut premiere air launch establish introduce approve nominate appoint succeed replace kill murder +retire resign own acquire merge sponsor manage train perform tour reach finish rank place qualify earn receive""".split()) + + +def canon(w): + w = w.lower() + return IRREG.get(w) or stem(w) + + +def verbish(w): + wl = w.lower() + return wl in IRREG or wl in BASE_VERBS or (wl.endswith("ed") and len(wl) > 4) or stem(wl) in {stem(b) for b in BASE_VERBS} + + +def stem(w): + w = w.lower() + for suf in ("ing", "ers", "ed", "es", "er", "s"): + if len(w) > len(suf) + 3 and w.endswith(suf): + return w[: -len(suf)][:6] + return w[:6] + + +class CubeReader: + def __init__(self, path=DEEP): + uri = "file:%s?mode=ro&immutable=1" % path + self.db = sqlite3.connect(uri, uri=True, check_same_thread=False) + self._lead = {} + + # ---- K: the store ---- + def pid_of(self, title): + nt = norm(title) + if not nt: + return None + r = self.db.execute("SELECT page_id FROM title_alias WHERE normalized_title=? LIMIT 1", (nt,)).fetchone() + return r[0] if r else None + + def page(self, pid, cap=40): + out = [] + for (rowid,) in self.db.execute("SELECT source_rowid FROM page_sentence WHERE page_id=? ORDER BY sentence_ordinal " + "LIMIT ?", (pid, cap)).fetchall(): + c = self.db.execute("SELECT c0, c1 FROM source_rows_content WHERE id=?", (rowid,)).fetchone() + if not c: + continue + text = " ".join(str(c[1]).split()) + # the store's sentence splitter breaks on initials ("discovered by A." / "Schwassmann on 11 September 1898"): + # rejoin, keeping the first claim id and citing both + if out and re.search(r"(?:^|\s)[A-Z]\.$", out[-1][1]): + out[-1] = (out[-1][0], out[-1][1] + " " + text) + else: + out.append((c[0], text)) + return out + + def lead(self, pid, n=3): + if pid not in self._lead: + self._lead[pid] = " ".join(t for _, t in self.page(pid, n)) + return self._lead[pid] + + # ---- Q: FRAME ---- + def frame(self, q): + s = " ".join(str(q).split()) + low = s.lower() + if not s or "passage:" in low or "\noptions" in low or "options:" in low or re.search(r"\b(you|your)\b", low): + return None + if not (s.endswith("?") or re.match(r"^(who|whom|whose|what|which|when|where|how|in what|in which|name)\b", low)): + return None + atype, head = None, None + if re.search(r"\bwhat (day|date)\b|\bday, month,? and year\b|\bexact date\b|\b(what|which) month and year\b", low): + atype = "DATE" + elif re.search(r"\b(what|which) year\b|\bin what year\b|^when\b|\bwhat year\b", low): + atype = "YEAR" + elif re.search(r"\bhow many\b", low): + atype = "NUMBER" + elif re.match(r"^(who|whom)\b", low) or re.search(r"\b(which|what) (person|man|woman|actor|actress|singer|player|" + r"writer|author|politician|artist|director)\b", low): + atype = "PERSON" + elif re.match(r"^where\b", low): + atype, head = "PLACE", None + else: + m = re.search(r"\b(?:which|what)\s+((?:[\w'\-]+\s+){0,5}?[\w'\-]+?)(?=\s+(?:is|was|were|are|did|does|do|has|had|" + r"have|won|played|made|became|served|wrote|directed|founded|released|located|called|named|known|" + r"used|held|given|awarded|built|born|died|developed|invented|created|published|produced|starred|" + r"in|of|for|by|to|at|on|from|with|that|whose|who|which|as|after|before|during)\b|\s*\?|$)", s, re.I) + if m: + words = [w for w in m.group(1).split() if w] + lower = [w for w in words if w.islower() and w.isalpha()] + if lower: + head = lower[-1].lower() + if head in FRAME_NOUNS: + m2 = re.search(r"\b(?:name|type|kind|sort|title) of (?:the |a |an )?((?:[\w'\-]+\s+){0,3}?[\w'\-]+?)" + r"(?=\s+(?:in|of|for|by|that|which|who|whose|is|was|were)\b|\s*\?|$)", s, re.I) + head = ([w for w in m2.group(1).split() if w.islower()] or [None])[-1] if m2 else None + if head in MEASURES: + atype = "NUMBER" + elif head in ("city", "country", "town", "state", "place", "capital", "village", "nation", "province"): + atype = "PLACE" + elif head and head not in STOP and len(head) >= 3: + atype = "INSTANCE" + if atype is None: + return None + if COMPARE.search(s) and (" or " in low or " out of " in low or " between " in low or " both " in low): + return None # a comparison: the COMPARE move is Phase 3, not this reader + if re.search(r"\bthe (?:[a-z\-]+ ){0,2}(?:man|woman|person|composer|author|writer|director|actor|actress|singer|" + r"player|team|band|company|film|city|country|village|town|artist|founder|creator|leader|president|" + r"king|queen|school|university|group|album|show|series|book|novel|song|character|state|river)\s+" + r"(?:who|whose|which|that|where)\b", low): + return None # the subject is a DESCRIPTION: a bridge (two hops), Phase 3 + if re.search(r"\bthe\b[^?]{0,90}?,?\s+(?:who|whose)\b", low): + return None # v1.2: any described person ("the American lawyer ... who") is a bridge + if re.search(r"\b(?:of|on|in|at|from|near)\s+the\s+(?:[a-z\-]+\s+){0,2}[a-z]+\s+(?:located|situated|based|founded|" + r"built|named|called|known|found|owned|operated|run|held)\b", low): + return None # v1.3: "the slopes of the hill located in what town" - the + # asked thing is a described entity (a bridge), not the subject + return {"q": s, "atype": atype, "head": head, "years": set(YEAR.findall(s)), + "month_year": bool(re.search(r"\bmonth and year\b|\bmonth, and year\b", low)) and "day" not in low} + + # ---- O: LINK (title index, longest non-overlapping n-grams) ---- + def link(self, fr, maxn=7): + toks = TOK.findall(fr["q"]) + found = [] + for n in range(min(maxn, len(toks)), 0, -1): + for i in range(len(toks) - n + 1): + words = toks[i:i + n] + nt = norm(" ".join(words)) + if not nt or all(w in STOP or w in FRAME_NOUNS for w in nt.split()) or nt == fr["head"]: + continue + if not words[0][:1].isupper() and not words[0][:1].isdigit(): + continue # a subject starts with a capital (names, titles) or a digit + if n == 1 and len(nt) < 4: + continue + if nt in MONTHS or nt.replace(" ", "").isdigit(): + continue # a month or a bare number is a constraint, never the subject + pid = self.pid_of(nt) + if pid: + found.append((i, i + n, " ".join(words), pid)) + keep, used = [], set() + for i, j, text, pid in sorted(found, key=lambda t: (-(t[1] - t[0]), t[0])): + if used.isdisjoint(range(i, j)): + keep.append({"span": (i, j), "text": text, "pid": pid}) + used.update(range(i, j)) + fr["tokens"] = toks + fr["subjects"] = keep + subj_tok = set(k for s in keep for k in range(*s["span"])) + rel = [t for k, t in enumerate(toks) if k not in subj_tok and t.lower() not in STOP and t.lower() != fr["head"] + and t.lower() not in ANSWER_WORDS and not YEAR.fullmatch(t) and t[:1].islower() and len(t) >= 3] + fr["rel"] = [stem(t) for t in rel] + # the PREDICATE: the question's verb(s). The binding sentence must carry one of them - a value bound through a + # shared context word ("road" in "road running athlete") answers a different relation + verbs = [(k, t) for k, t in enumerate(toks) if k not in subj_tok and t[:1].islower() and verbish(t) + and t.lower() not in STOP] + # only the verb GOVERNING the answer slot counts: the one nearest the wh-phrase ("founded in what year"), not a + # participle describing something else ("a failed raid") + wh = [k for k, t in enumerate(toks) if t.lower() in ("who", "whom", "whose", "what", "which", "when", "where", "how")] + if verbs and wh: + w0 = wh[0] + k, t = min(verbs, key=lambda kt: (abs(kt[0] - w0), kt[0] < w0)) + fr["pred"] = [canon(t)] + else: + fr["pred"] = sorted({canon(t) for _, t in verbs}) + # every NAME in the question is an obligation: a capitalised token (not the first word) outside every linked span + unlinked = [t for k, t in enumerate(toks) if k > 0 and k not in subj_tok and t[:1].isupper() + and t.lower() not in STOP and len(t) >= 3] + unlinked += [t for k, t in enumerate(toks) if k not in subj_tok and t.isdigit() and len(t) >= 2 and not YEAR.fullmatch(t)] + # v1.2: an identifier ("gal56", "EURECA-1") is an obligation too: letters and digits together + unlinked += [t for k, t in enumerate(toks) if k not in subj_tok and re.search(r"\d", t) and re.search(r"[A-Za-z]", t)] + # v1.2: the ROLE the answer must play. "Who beat X" asks for the AGENT of the predicate + # v1.3: "Who did X knock out?" asks for the PATIENT of X's action + fr["patient"] = bool(re.match(r"^(?:who|whom|what|which [\w\-]+)\s+(?:did|does|do|has|had|have)\b", fr["q"].lower())) + fr["agent"] = bool(re.match(r"^(?:who|which [\w\-]+(?: [\w\-]+)?)\s+(?!is\b|was\b|are\b|were\b|did\b|does\b|do\b|has\b|" + r"had\b|have\b|will\b|would\b|can\b|could\b)[a-z]+", fr["q"].lower())) + fr["constraints"] = [s["text"] for s in keep[1:]] + sorted(fr["years"]) + unlinked + return keep + + # ---- K: BIND ---- + def _candidates(self, fr, sent, qnorm): + at = fr["atype"] + out = [] + if at == "YEAR": + out = [y for y in YEAR.findall(sent) if y not in fr["years"]] + elif at == "DATE": + out = DATE.findall(sent) + if not out and fr.get("month_year"): + out = MONTH_YEAR.findall(sent) + elif at == "NUMBER": + out = [n for n in NUMBER.findall(sent) if not YEAR.fullmatch(n) and n not in fr["q"]] + else: + for m in CAPSPAN.finditer(sent): + span = m.group(1).strip(" .,;:") + span = re.sub(r"^(The|A|An)\s+", "", span) + if len(span) < 3 or (span.isupper() and len(span) <= 3) or norm(span) in qnorm or norm(span).split()[0] in STOP: + continue + if span.split()[0].capitalize() in MONTH_RE.split("|"): + continue + out.append(span) + return out + + def bind(self, fr): + qnorm = norm(fr["q"]) + rel = set(fr["rel"]) + cands = [] + for subj in fr["subjects"][:4]: + sents = self.page(subj["pid"], 40) + for idx, (cid, sent_full) in enumerate(sents): + words = [p for t in TOK.findall(sent_full) for p in [t] + t.split("-")[1:]] # "co-founded" -> founded + # v1.2: the predicate must appear as a VERB - "composers" is not "composed" + verbs_here = {canon(t) for t in words if not NOUNISH.search(t)} + if fr["pred"] and not (verbs_here & set(fr["pred"])): + continue # the question's predicate is not in this sentence + sent = sent_full + if not ({"bear", "die"} & set(fr["pred"])): + # a value inside "(born ...)" or a life span answers only a birth/death question + sent = re.sub(r"\((?:[^()]*\b(?:born|died|b\.|d\.)\b[^()]*|\s*[^()]{0,30}\d{3,4}\s*[–-]\s*[^()]{0,30}\d{3,4}\s*)\)", + "", sent_full) + stoks = [stem(t) for t in TOK.findall(sent)] + hits = len(rel & set(stoks)) + cover = (hits / len(rel)) if rel else (1.0 if idx < 2 else 0.0) + if rel and hits == 0: + continue # the sentence must carry at least one of the question's relation words + if not rel and idx >= 2: + continue # no relation words: only the page's lead may bind + for c in self._candidates(fr, sent, qnorm): + # proximity: the candidate sits near a relation word in its sentence + pos = sent.find(c) + near = any(abs(sent.lower().find(r) - pos) < 90 for r in rel if sent.lower().find(r) >= 0) if rel else True + if not near: + continue + cands.append({"value": c, "claim": cid, "sentence": sent_full, "subject": subj, "cover": cover, + "lead": idx < 3, "score": cover + (0.3 if idx < 3 else 0.0)}) + return cands + + # ---- O: CHECK (independent of the proposing sentence) ---- + def check(self, fr, c): + at = fr["atype"] + ev = {} + if at in ("PERSON", "PLACE", "INSTANCE"): + pid = self.pid_of(c["value"]) + if not pid or pid == c["subject"]["pid"]: + return False, {"why": "candidate has no page of its own"} + lead = self.lead(pid) + if at == "PERSON" and not PERSON_LEAD.search(lead): + return False, {"why": "candidate page does not describe a person"} + first = re.split(r"(?<=[.!?])\s+", lead, 1)[0] + if at == "PLACE" and not fr["head"] and not PLACE_LEAD.search(first): + return False, {"why": "candidate page does not describe a place"} + if at == "INSTANCE" or (at == "PLACE" and fr["head"]): + # IS-A, not mention: the first sentence must say the candidate IS a ("... is a professional team") + h = fr["head"] + # ... and the head noun must END its phrase ("a film", not "a film producer") + isa = (r"\b(?:is|was|are|were)\s+(?:a|an|the|one of the)\b[^.]{0,90}?\b%s(?:s|es)?\b" + r"(?=\s*(?:[,.;:()]|$)|\s+(?:that|which|who|whose|based|in|of|from|and|or|founded|established|located|" + r"with|by|for|released|directed|produced|written|set|created|developed|published|made|first|since|" + r"owned|operated|consisting|starring|about|on|at|to|named|known|situated|headquartered)\b)") % re.escape(h) + # the store's canonical title is also type evidence when its last word IS the head ("Crimean War") + title_head = norm(c["value"]).split()[-1:] in ([h], [h + "s"]) + if not re.search(isa, first, re.I) and not title_head: + return False, {"why": "candidate page does not say it is a %s" % h} + if fr.get("agent"): + # v1.2 ROLE: "Who beat X?" needs the candidate as the predicate's AGENT. In active voice the agent stands + # before the verb; after it, only a passive "by" makes it the agent ("discovered by Annibale de Gasparis"). + # "Henman ... beating Roger Federer" puts Federer in the patient role and is refused. + sl = c["sentence"] + pos_c = sl.find(c["value"]) + pv = next((m for m in TOK.finditer(sl) if not NOUNISH.search(m.group(0)) and canon(m.group(0)) in fr["pred"]), None) + if pv is not None and pos_c > pv.start() and not re.search(r"\bby\b", sl[pv.end():pos_c]): + return False, {"why": "candidate is not the agent of the predicate"} + if fr.get("patient"): + # v1.3 ROLE: the answer is the patient of the SUBJECT's action; a sentence where the subject is itself + # the passive patient ("Douglas was knocked out ... Lou Savarese") names the agent, not the answer + subj_words = [w for w in norm(c["subject"]["text"]).split() if len(w) >= 4] + sl = c["sentence"] + for w in subj_words: + if re.search(r"\b%s\b[^.]{0,40}?\b(?:was|were|been|being)\s+(?:\w+\s+)?\w+(?:ed|en|out)\b" % re.escape(w), sl, re.I): + return False, {"why": "the subject is the patient in this sentence"} + ev = {"type_page": pid, "type_lead": lead[:240]} + where = norm(c["sentence"] + " " + lead + " " + self.lead(c["subject"]["pid"])) + else: + if at == "DATE" and not fr.get("month_year"): + # v1.3 CORROBORATION for day-precision dates: a single sentence can be wrong (FC Baltika: the store says + # 23 August 1954, the key 22 December 1954). An exact date is committed only when a SECOND sentence - + # different claim id - carries the same date string with the subject's distinctive word. + subj_words = [w for w in norm(c["subject"]["text"]).split() if len(w) >= 4 and w not in STOP] + word = max(subj_words, key=len) if subj_words else None + n_other = 0 + if word: + try: + rows = self.db.execute("SELECT claim_id FROM source_rows WHERE source_rows MATCH ? LIMIT 6", + ('"%s" AND "%s"' % (c["value"].replace('"', ""), word),)).fetchall() + n_other = len({r[0] for r in rows if r[0] != c["claim"]}) + except sqlite3.OperationalError: + n_other = 0 + if n_other < 1: + return False, {"why": "exact date not corroborated by a second sentence"} + vals = self._candidates(fr, c["sentence"], norm(fr["q"])) + if len(set(vals)) != 1: + return False, {"why": "more than one %s in the binding sentence" % at.lower()} + where = norm(c["sentence"] + " " + self.lead(c["subject"]["pid"])) + # constraints: every OTHER name/number/year in the question. The subject whose page holds the sentence is attested + # by the page itself; qualifiers of that subject ("El Santuario, Antioquia, Colombia") may be attested on its lead; + # a person's name is attested by its surname + own = norm(c["subject"]["text"]) + cons = [x for x in fr["constraints"] if norm(x) and norm(x) != own and norm(x) not in own] + linked_other = {norm(s["text"]) for s in fr["subjects"] if norm(s["text"]) != own} + sent_n = norm(c["sentence"]) + missing = [] + for x in cons: + nx = norm(x) + sur = nx.split()[-1] if len(nx.split()) > 1 and len(nx.split()[-1]) >= 4 else None + # another NAMED entity of the question must be in the binding sentence itself (not merely on a lead): the + # prize page's "awarded since 1979" says nothing about the person the question asks about + # v1.3: a YEAR in the question qualifies the event, so it must be in the binding sentence too + scope = sent_n if (nx in linked_other or YEAR.fullmatch(x.strip())) else where + if nx not in scope and not (sur and (" %s " % sur) in (" %s " % scope)): + missing.append(x) + if missing: + return False, {"why": "constraint not attested: %s" % missing} + return True, ev + + # ---- Q: VERIFY + the whole cycle ---- + def answer(self, q): + self.stop = "frame" + fr = self.frame(q) + if not fr: + return None + self.stop = "link" + if not self.link(fr): + return None + self.stop = "bind" + cands = self.bind(fr) + if not cands: + return None + self.stop = "check" + best = {} + for c in cands: + k = norm(c["value"]) + if k not in best or c["score"] > best[k]["score"]: + best[k] = c + ranked = sorted(best.values(), key=lambda c: -c["score"]) + passed = [] + self.why = [] + for c in ranked[:6]: + ok, ev = self.check(fr, c) + if ok: + c["check"] = ev + passed.append(c) + else: + self.why.append((c["value"], ev.get("why"))) + if not passed: + return None + top = passed[0] + if len(passed) > 1 and passed[1]["score"] >= top["score"] - 1e-9: + self.stop = "tie" + return None # two checked candidates tie: nothing is said + self.stop = "answered" + return {"answer": top["value"], "claim": top["claim"], "sentence": top["sentence"], "subject": top["subject"]["text"], + "type": fr["atype"], "head": fr["head"], "check": top["check"], "frame": {k: fr[k] for k in ("atype", "head", "rel")}} diff --git a/root/wander-train/oracle_game_combine_v1.py b/root/wander-train/oracle_game_combine_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..01a6265ee6272c105c7ceda6334a1bdf2954db40 --- /dev/null +++ b/root/wander-train/oracle_game_combine_v1.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""ORACLE GAME COMBINATORS - operators that take games and return games. + +Twelve categories with parameters is a large space. It is still a flat one: every +game it can build is one of twelve shapes wearing different numbers. The move that +makes it genuinely open is HIGHER ORDER operators, which take a game and give back +a different game, because then the space closes under its own operations. + +Two of them, and they are chosen because both are real games people actually play +rather than formal curiosities: + + INVERT the win condition is flipped. Get three in a row and you LOSE, + which is a real game (misere noughts and crosses) with completely + different play from the one it inverts. Applied to any of the + twelve, not just to take-away where it was hand-written. + + COMPOUND play game A, then game B, and whoever won more takes the match. + "Tic tac toe then nim" is not in any catalogue, is not a + parameter setting of anything, and is a perfectly well defined + game. This is what combining categories means. + +Both come with their own referees, built by composing the referees underneath +them, so a combined game is checked the same way a simple one is: the engine +plays, the referee replays, and the emitted program runs in a separate process. +A combination is only allowed out if all three agree about it. + +The count matters less than the closure. Twelve categories give twelve games; with +INVERT that is twenty-four; with COMPOUND over ordered pairs it is 24 x 24 before +any parameter is chosen, and the parameters are still free. Nothing in that space +is stored. +""" + +from __future__ import annotations + +import re + +from oracle_game_kernel_v1 import GameError, play + +COMPOUND_BUDGET = 1200 +from oracle_game_vocab_v1 import CATS + +_FLIP = {'first': 'second', 'second': 'first', 'draw': 'draw', + 'solved': 'failed', 'failed': 'solved'} + + +# ── INVERT ─────────────────────────────────────────────────────────────────── +def invert_build(inner_name, params): + inner = CATS[inner_name]['build'](params) + base_outcome = inner['outcome'] + + def outcome(st): + got = base_outcome(st) + return _FLIP[got] if got is not None else None + + return dict(inner, outcome=outcome) + + +def invert_referee(inner_name, params, log): + got = CATS[inner_name]['referee'](params, log) + return _FLIP[got] + + +# ── COMPOUND ───────────────────────────────────────────────────────────────── +def compound_build(a_name, b_name, a_params, b_params): + """Play A to a result, then B, then compare. One game, two halves.""" + ga = CATS[a_name]['build'](a_params) + gb = CATS[b_name]['build'](b_params) + + def score_of(res): + return {'first': 1, 'solved': 1, 'second': -1, 'failed': -1, + 'draw': 0}[res] + + # Every move is TAGGED with the half it belongs to, so the kernel's own move + # log is already splittable by the referee. The first version kept a private + # tagged log inside the state and handed the referee the kernel's untagged + # one, which is why every compound died on unpacking. + def init(): + return ('a', ga['init'](), None, 0) + + def moves(st): + leg, inner = st[0], st[1] + g = ga if leg == 'a' else gb + legal = [(leg, m) for m in g['moves'](inner)] + if not legal and leg == 'a': + # the first half ran out of legal moves without declaring a result. + # Returning nothing here makes the KERNEL call the whole compound a + # draw and the second half never happens, which the referee then + # rightly refuses. Hand back a marker so the transition still occurs. + return [('a', None)] + return legal + + def apply(st, tagged, t): + leg, inner, first_res, count = st + _tag, m = tagged + if m is None: # the exhaustion marker from moves() + return ('b', gb['init'](), ga['outcome'](inner) or 'draw', 0) + g = ga if leg == 'a' else gb + inner2 = g['apply'](inner, m, count) # the INNER half's own turn + done = g['outcome'](inner2) + if done is not None and leg == 'a': + return ('b', gb['init'](), done, 0) + return (leg, inner2, first_res, count + 1) + + def outcome(st): + leg, inner, first_res, _count = st + if leg == 'a': + return None + done = gb['outcome'](inner) + if done is None: + return None + total = score_of(first_res) + score_of(done) + return 'first' if total > 0 else 'second' if total < 0 else 'draw' + + def policy(st, legal, t): + return legal[0] if st[3] % 2 == 0 else legal[-1] + + return {'init': init, 'moves': moves, 'apply': apply, 'outcome': outcome, + 'policy': policy} + + +def compound_referee(a_name, b_name, a_params, b_params, log): + """Split the tagged log and hand each half to the referee that owns it.""" + a_log = [m for leg, m in log if leg == 'a' and m is not None] + b_log = [m for leg, m in log if leg == 'b' and m is not None] + if not a_log or not b_log: + raise GameError('referee: one half of the compound never happened') + ra = CATS[a_name]['referee'](a_params, a_log) + rb = CATS[b_name]['referee'](b_params, b_log) + sc = {'first': 1, 'solved': 1, 'second': -1, 'failed': -1, 'draw': 0} + total = sc[ra] + sc[rb] + return 'first' if total > 0 else 'second' if total < 0 else 'draw' + + +# ── routing the combinations ───────────────────────────────────────────────── +_INVERT_CUE = re.compile( + r'\b(?:misere|mis[eè]re)\b|you lose|makes you lose|loses instead' + r'|whoever .{0,24}loses|reverse(?:d)? (?:rules|win)|inverted' + r'|the loser wins|last one loses', re.I) +_COMPOUND_CUE = re.compile( + r'\bthen\b|\bfollowed by\b|\band then\b|\bboth\b.{0,20}\bgames?\b' + r'|combin(?:e|ed|ation) of', re.I) + + +def find_categories(surface): + """Every category the surface names, in the order it names them.""" + hits = [] + for name, c in CATS.items(): + for rx in c['cues']: + m = rx.search(surface) + if m: + hits.append((m.start(), name)) + break + hits.sort() + out = [] + for _pos, name in hits: + if name not in out: + out.append(name) + return out + + +def parse_combination(surface): + """(kind, payload) or None if this is not a combination request. + + kind is 'invert' or 'compound'. Raises GameError when the request clearly + wants a combination but the pieces do not resolve, so it can decline with a + reason rather than quietly build one half. + """ + names = find_categories(surface) + wants_compound = bool(_COMPOUND_CUE.search(surface)) + wants_invert = bool(_INVERT_CUE.search(surface)) + + if wants_compound: + real = [n for n in names if n != 'misere_take_away'] + if len(real) < 2: + raise GameError('that asks for two games joined together and I can ' + 'only find one of them in it') + a, b = real[0], real[1] + pa = CATS[a]['params'](surface) + pb = CATS[b]['params'](surface) + return 'compound', (a, b, pa, pb) + + if wants_invert and names: + base = [n for n in names if n != 'misere_take_away'] + if not base: + return None # plain misere nim, already a category + n = base[0] + return 'invert', (n, CATS[n]['params'](surface)) + return None + + +def describe(kind, payload): + if kind == 'invert': + n, p = payload + return ('%s, with the win condition inverted: doing the thing that ' + 'normally wins LOSES' % CATS[n]['describe'](p)) + a, b, pa, pb = payload + return ('two games in one match: first %s, then %s, and whoever wins more ' + 'takes it' % (CATS[a]['describe'](pa), CATS[b]['describe'](pb))) + + +def build_and_referee(kind, payload): + if kind == 'invert': + n, p = payload + return (invert_build(n, p), + lambda _params, log: invert_referee(n, p, log), p) + a, b, pa, pb = payload + return (compound_build(a, b, pa, pb), + lambda _params, log: compound_referee(a, b, pa, pb, log), + {'a': a, 'b': b}) diff --git a/root/wander-train/oracle_program_ir_v1.py b/root/wander-train/oracle_program_ir_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..9dea06ba74265247ea6ca17cf80fadeaa27ff2bb --- /dev/null +++ b/root/wander-train/oracle_program_ir_v1.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""oracle_program_ir_v1 - one intermediate representation; every request compiles to a bounded relational program. + +The five things that looked like separate engineering tracks - corpus reach, coding, conversation, prose, +self-repair - are one missing operation. A request must compile into explicit semantic OBLIGATIONS, those +obligations into a bounded relational PROGRAM over stores, operators, discourse state and permitted inference, +and the completed result must be verified AGAINST THE OBLIGATIONS before anything is rendered. + +I found this while working on coding and then confined it to coding. The coding fold says: the request contains +its own specification, and testing a program against its own behaviour verifies the wrong proposition. Every other +capability makes the identical error. Asked why a compiler needs a parser, the model checks whether retrieved +sentences match the surface wording. Asked whether oxygen is a cause or a result, it checks a taxonomic exclusion. +Asked to compare photosynthesis with respiration, it retrieves individually warranted fragments and loses the +comparison subject. In each case the local claims are valid and the executed answer is not bound to what was +asked - the same failure as returning a bank-account class for a stack and reporting that it executed. + +THE CONSEQUENCE THAT MATTERS MOST: an answer need not exist as a stored sentence. It exists as a warranted path +through stored relations. No sentence anywhere says why a compiler needs a parser; the answer is a join across +what a compiler consumes and what a parser produces. Measuring whether the answer STRING is in the store - which +is what every coverage probe here has done, mine included - can only ever report a coverage problem. That is the +difference between possessing documents and possessing knowledge. + + utterance -> obligations -> relational program -> execution trace -> answer-state -> expression + +A successful answer is a verified execution trace satisfying every obligation, not a collection of valid sentences. + +This module is the representation and the obligation compiler. It executes nothing: an obligation it cannot build +is itself a finding, and is recorded as `unbuildable` rather than quietly dropped. + +usage: oracle_program_ir_v1.py [--packet PATH] compile the diagnostic prompts and print their programs +""" + +from __future__ import annotations + +import json +import re +import sys +from dataclasses import dataclass, field, asdict +from pathlib import Path + +WORD = re.compile(r"[A-Za-z][A-Za-z'-]{1,}") +PROPER = re.compile(r"\b([A-Z][\w'-]+(?:\s+(?:of|the|de)\s+[A-Z][\w'-]+|\s+[A-Z][\w'-]+)*)") +PRONOUN = re.compile(r"\b(it|its|that|this|they|them|those|these|one)\b", re.I) +NEGATION = re.compile(r"\b(not|isn't|aren't|doesn't|don't|never|no longer|rather than|instead of)\b", re.I) +PAST = re.compile(r"\b(was|were|did|had|used to|formerly|originally)\b", re.I) +QUANTIFIER = re.compile(r"\b(all|every|each|some|any|most|three|two|both|several|many|few|\d+)\b", re.I) +# A capital letter at the start of a sentence is not a name. Taking "What", "Compare", "Is" as entities is the +# same head-word disease this whole line exists to fix, and the first version of this file had it. +NOT_A_NAME = {'what', 'which', 'who', 'when', 'where', 'why', 'how', 'is', 'are', 'was', 'were', 'do', 'does', + 'did', 'can', 'could', 'will', 'would', 'should', 'tell', 'give', 'list', 'name', 'compare', + 'explain', 'describe', 'summarize', 'summarise', 'correct', 'the', 'a', 'an', 'so', 'and', 'but', + 'if', 'that', 'this', 'it', 'they', 'in', 'on', 'at', 'for', 'write', 'solve', 'find'} +# word-bounded and applied to the LOWERCASED utterance (fixed 2026-09-23): unbounded, it matched the tail of every +# capitalised word, so "Summarize the Internet" yielded the subjects 'ummarize' and 'nternet' +CONTENT = re.compile(r"\b([a-z][a-z'-]{3,})\b") +FUNCTION = {'about', 'from', 'with', 'into', 'than', 'then', 'have', 'been', 'does', 'made', 'more', 'most', + 'much', 'many', 'some', 'such', 'like', 'also', 'just', 'over', 'under', 'between', 'because', + 'there', 'here', 'them', 'their', 'your', 'mine', 'ours', 'wrong', 'right', 'mean', 'means'} + + +@dataclass +class Obligation: + """something the answer must satisfy to BE an answer. Verification is against these, never against fragments.""" + name: str + kind: str # resolve | retrieve | relate | join | compute | contrast | decide | explain + argument: str = '' + satisfied_by: str = '' # filled at execution: what satisfied it + satisfied: bool | None = None # None until executed + note: str = '' + + +@dataclass +class Request: + """the intermediate representation. One shape; the program TYPE varies over it.""" + utterance: str + program_type: str = 'unknown' + entities: list = field(default_factory=list) + discourse_referents: list = field(default_factory=list) # pronouns needing binding from prior turns + requested_relation: str = '' + requested_operation: str = '' + quantifiers: list = field(default_factory=list) + polarity: str = 'positive' + tense: str = 'present' + obligations: list = field(default_factory=list) + completion_criteria: list = field(default_factory=list) + content_subjects: list = field(default_factory=list) + unbuildable: list = field(default_factory=list) + + +# Each program type declares the obligations its answers must satisfy. These are the SHAPES of questions, and a +# kernel slot is an executable question-shape rather than a prose template: definition, constitution, mechanism, +# contrast, criterion, limitation, consequence. +PROGRAM_TYPES = { + 'definition': ['resolve:subject', 'retrieve:genus', 'retrieve:differentia', 'explain:definition'], + 'constitution': ['resolve:subject', 'relate:part_of', 'explain:composition'], + 'mechanism': ['resolve:subject', 'relate:process', 'join:steps', 'explain:mechanism'], + 'cause': ['resolve:subject', 'relate:causal_edge', 'decide:cause_vs_result', 'explain:causation'], + 'dependence': ['resolve:dependent', 'resolve:dependency', 'retrieve:consumes', 'retrieve:produces', + 'join:through_intermediate', 'decide:dependency_follows', 'explain:why_needed'], + 'comparison': ['resolve:left', 'resolve:right', 'retrieve:left_properties', 'retrieve:right_properties', + 'contrast:shared_dimension', 'explain:difference'], + 'counterfactual': ['resolve:subject', 'relate:causal_edge', 'decide:antecedent_removed', + 'explain:consequence'], + 'enumeration': ['resolve:subject', 'retrieve:n_facts', 'decide:distinctness', 'explain:list'], + 'summarization': ['resolve:scope', 'retrieve:claims_in_scope', 'contrast:redundancy', 'explain:summary'], + 'correction': ['resolve:prior_claim', 'retrieve:contrary_evidence', 'decide:which_holds', + 'explain:correction'], + 'coding': ['resolve:specification', 'compute:program', 'decide:satisfies_specification', + 'explain:program'], + 'computation': ['resolve:operands', 'resolve:operation', 'compute:exact', 'explain:steps'], + 'argumentation': ['resolve:thesis', 'retrieve:premises', 'join:dependency_graph', 'contrast:objections', + 'decide:thesis_supported', 'explain:argument'], +} + +# surface cues -> program type. Deliberately small and readable: this is a claim about which question is being +# asked, and a wrong claim here should be legible in the trace rather than buried. +CUES = [ + ('dependence', re.compile(r"\bwhy (?:does|do|is|are|would).{0,40}\b(?:need|require|depend|use)\b", re.I)), + ('dependence', re.compile(r"\b(?:need|require|depend on)\b.{0,30}\?", re.I)), + ('counterfactual', re.compile(r"\bwhat if\b|\bwould happen if\b|\bhad .{0,20} not\b|\bwithout\b.{0,30}\?", re.I)), + ('cause', re.compile(r"\bwhat causes?\b|\bwhy (?:does|do|is|are)\b|\bcause or (?:a )?result\b", re.I)), + ('comparison', re.compile(r"\bcompare\b|\bdifference between\b|\bhow is .{0,30} different\b|\bversus\b|\bvs\.?\b", re.I)), + ('correction', re.compile(r"\bcorrect me\b|\bthat'?s wrong\b|\bactually,?\b|\bisn'?t it\b", re.I)), + ('summarization', re.compile(r"\bsummari[sz]e\b|\bwhat have we\b|\bso far\b|\brecap\b", re.I)), + ('enumeration', re.compile(r"\blist\b|\bname (?:three|two|\d+)\b|\bgive me (?:three|two|\d+)\b", re.I)), + ('mechanism', re.compile(r"\bhow does\b|\bhow do\b|\bhow is .{0,20} made\b|\bwork\b\s*\?", re.I)), + ('constitution', re.compile(r"\bmade of\b|\bcomposed of\b|\bparts of\b|\bconsists? of\b", re.I)), + ('computation', re.compile(r"\d.{0,20}(?:%|percent|times|plus|minus|divided)|\bsquare root\b|\bsolve for\b", re.I)), + ('coding', re.compile(r"\bwrite a (?:python|javascript|java|c|go|sql)\b|\bfunction that\b|\bclass for\b", re.I)), + ('definition', re.compile(r"^\s*what (?:is|are|was|were)\b|\btell me about\b|\bwhat'?s\b", re.I)), +] + + +def classify(utterance): + for name, rx in CUES: + if rx.search(utterance or ''): + return name + return 'unknown' + + +def compile_request(utterance, discourse_topics=()): + """utterance -> Request with its obligations. Executes nothing.""" + u = ' '.join(str(utterance or '').split()) + r = Request(utterance=u) + r.program_type = classify(u) + # names, minus anything that is only capitalised because it began the sentence + r.entities = [m.group(1) for m in PROPER.finditer(u) + if m.group(1).split()[0].lower() not in NOT_A_NAME] + # lowercase content nouns are subjects too - "compare photosynthesis with respiration" names two things and + # capitalises neither + r.content_subjects = [w for w in CONTENT.findall(u.lower()) if w not in FUNCTION and w not in NOT_A_NAME] + r.discourse_referents = sorted({m.group(1).lower() for m in PRONOUN.finditer(u)}) + r.quantifiers = sorted({m.group(1).lower() for m in QUANTIFIER.finditer(u)}) + r.polarity = 'negative' if NEGATION.search(u) else 'positive' + r.tense = 'past' if PAST.search(u) else 'present' + + # the requested relation is what the answer must ESTABLISH, not what the words resemble + if r.program_type == 'dependence': + r.requested_relation = 'functional_dependence' + elif r.program_type in ('cause', 'counterfactual'): + r.requested_relation = 'causal' + elif r.program_type == 'comparison': + r.requested_relation = 'contrast_on_shared_dimension' + elif r.program_type in ('definition', 'constitution'): + r.requested_relation = 'taxonomic' + r.requested_operation = r.program_type + + template = PROGRAM_TYPES.get(r.program_type) + if not template: + r.unbuildable.append('no program type recognised for this utterance') + return r + + # the SUBJECTS of the program: named entities, else the discourse referents that must be bound from prior turns + subjects = list(r.entities) + [c for c in r.content_subjects if c not in + {e.lower() for e in r.entities}] + for step in template: + kind, _, arg = step.partition(':') + ob = Obligation(name=step, kind=kind, argument=arg) + if kind == 'resolve': + if r.discourse_referents and not subjects: + # THE FAILURE THE TRACES KEEP SHOWING: the act is identified and its subject is a pronoun that + # nothing binds. Recorded as an obligation with a named source, not as a dead end. + ob.note = 'subject is a discourse referent (%s); must bind from prior turns' % ', '.join(r.discourse_referents) + elif subjects: + ob.note = 'candidate: %s' % subjects[0] + else: + ob.note = 'no subject available in the utterance' + r.obligations.append(ob) + + r.completion_criteria = ['every obligation satisfied', + 'the established relation is the one requested (%s)' % (r.requested_relation or 'n/a'), + 'each claim carries a warrant', + 'no two admitted claims contradict'] + if r.program_type == 'comparison' and len(subjects) < 2: + r.unbuildable.append('comparison needs two subjects; found %d' % len(subjects)) + return r + + +def verify(request, trace): + """check the completed trace against the OBLIGATIONS - never against the fragments it produced.""" + got = {str(k) for k in (trace or {}).keys()} + unmet = [] + for ob in request.obligations: + ob.satisfied = ob.name in got + ob.satisfied_by = str((trace or {}).get(ob.name, ''))[:120] + if not ob.satisfied: + unmet.append(ob.name) + return {'satisfied': len(request.obligations) - len(unmet), 'total': len(request.obligations), + 'unmet': unmet, 'answerable': not unmet} + + +def main(): + a = sys.argv[1:] + packet = a[a.index('--packet') + 1] if '--packet' in a else '/root/fresh_conv_test.json' + prompts = [] + try: + d = json.loads(Path(packet).read_text(encoding='utf-8')) + for name, turns in d.items(): + for t in (turns if isinstance(turns, list) else []): + q = t[0] if isinstance(t, (list, tuple)) and t else (t if isinstance(t, str) else '') + if q: + prompts.append((name, q)) + except Exception as exc: # noqa: BLE001 + print('packet unreadable (%s); using built-in examples' % exc) + prompts = [('example', 'Why does a compiler need a parser?'), + ('example', 'Is oxygen a cause or a result of photosynthesis?'), + ('example', 'Compare photosynthesis with respiration.'), + ('example', 'List three facts about it.')] + + print('%d prompts\n' % len(prompts)) + unknown = 0 + unbound = 0 + for name, q in prompts: + r = compile_request(q) + if r.program_type == 'unknown': + unknown += 1 + needs_binding = [o for o in r.obligations if 'discourse referent' in o.note] + if needs_binding: + unbound += 1 + print('%-14s %-46s -> %s' % (name, q[:44], r.program_type)) + print(' relation : %s | entities: %s | referents: %s' + % (r.requested_relation or '-', ', '.join(r.entities[:3]) or '-', + ', '.join(r.discourse_referents) or '-')) + print(' program : %s' % ' -> '.join(o.name for o in r.obligations)) + for o in needs_binding: + print(' UNBOUND : %s (%s)' % (o.name, o.note)) + for u in r.unbuildable: + print(' UNBUILDABLE: %s' % u) + print() + print('%d prompts | %d had no recognised program type | %d have an unbound subject' + % (len(prompts), unknown, unbound)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/root/wander-train/oracle_release_conceptnet.py b/root/wander-train/oracle_release_conceptnet.py new file mode 100644 index 0000000000000000000000000000000000000000..05378e7e683102e0fd933c25b36c9ed741da11c1 --- /dev/null +++ b/root/wander-train/oracle_release_conceptnet.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""R1 — ConceptNet commonsense adapter for the release runtime. + +Typed edge retrieval over the sealed M13e5 ConceptNet specialist +(m13e5_conceptnet_build_a_v1, index sha 5e28406c...), rendered with provenance. +Zero-model: SQL + deterministic rendering only. Speech uses ConceptNet's own +crowd-authored surface_text where present (attributed), otherwise a typed +relation template. Every released line carries its edge provenance; empty +retrieval is an explicit abstention, never a guess. +""" + +from __future__ import annotations + +import re +import sqlite3 +from pathlib import Path +from typing import Any + +PHASE = Path(__file__).resolve().parents[1] / "fable-content" / "oracle-m-series-2026-08-15" +DEFAULT_BUILD = PHASE / "m13e5_conceptnet_build_a_v1" / "conceptnet_specialist.sqlite3" + +# Relation -> (question forms it answers, deterministic template) +RELATION_TEMPLATES: dict[str, str] = { + "IsA": "{start} is a kind of {end}", + "PartOf": "{start} is part of {end}", + "HasA": "{start} has {end}", + "UsedFor": "{start} is used for {end}", + "CapableOf": "{start} can {end}", + "AtLocation": "{start} is typically found at {end}", + "Causes": "{start} causes {end}", + "HasSubevent": "{start} involves {end}", + "HasPrerequisite": "{start} requires {end}", + "MotivatedByGoal": "{start} is motivated by {end}", + "Desires": "{start} desires {end}", + "MadeOf": "{start} is made of {end}", + "ReceivesAction": "{start} can be {end}", + "HasProperty": "{start} is {end}", + "Synonym": "{start} is a synonym of {end}", + "Antonym": "{start} is an antonym of {end}", + "DistinctFrom": "{start} is distinct from {end}", + "SimilarTo": "{start} is similar to {end}", + "RelatedTo": "{start} is related to {end}", +} + +QUESTION_ROUTES: list[tuple[re.Pattern[str], list[str]]] = [ + (re.compile(r"^what is (?:a |an |the )?(.+?) used for\??$", re.I), ["UsedFor"]), + (re.compile(r"^what can (?:a |an |the )?(.+?) do\??$", re.I), ["CapableOf"]), + (re.compile(r"^where (?:would you |do you |can you )?find (?:a |an |the )?(.+?)\??$", re.I), ["AtLocation"]), + (re.compile(r"^where is (?:a |an |the )?(.+?) (?:usually |typically )?found\??$", re.I), ["AtLocation"]), + (re.compile(r"^what (?:does|do) (?:a |an |the )?(.+?) have\??$", re.I), ["HasA"]), + (re.compile(r"^what is (?:a |an |the )?(.+?) made of\??$", re.I), ["MadeOf"]), + (re.compile(r"^what causes (?:a |an |the )?(.+?)\??$", re.I), ["Causes:end"]), + (re.compile(r"^what (?:does|do) (?:a |an |the )?(.+?) cause\??$", re.I), ["Causes"]), + (re.compile(r"^what kind of thing is (?:a |an |the )?(.+?)\??$", re.I), ["IsA"]), + (re.compile(r"^what (?:is|are) (?:a |an |the )?(.+?) (?:a )?(?:kind|type|sort) of\??$", re.I), ["IsA"]), + (re.compile(r"^what is (?:a |an |the )?(.+?) part of\??$", re.I), ["PartOf"]), + (re.compile(r"^what (?:is|are) (?:the )?parts? of (?:a |an |the )?(.+?)\??$", re.I), ["PartOf:end", "HasA"]), + (re.compile(r"^what do(?:es)? (?:a |an |the )?(.+?) require\??$", re.I), ["HasPrerequisite"]), + (re.compile(r"^what is (?:a |an |the )?(.+?) similar to\??$", re.I), ["SimilarTo", "Synonym"]), + (re.compile(r"^what is (?:the )?opposite of (?:a |an |the )?(.+?)\??$", re.I), ["Antonym"]), +] + +_SURFACE_BRACKETS = re.compile(r"\[\[|\]\]") + + +class ConceptNetAdapter: + def __init__(self, build_path: Path | str = DEFAULT_BUILD): + build_path = Path(build_path) + if not build_path.is_file(): + raise FileNotFoundError(str(build_path)) + uri = build_path.resolve().as_uri() + "?mode=ro&immutable=1" + self.connection = sqlite3.connect(uri, uri=True) + + # ── retrieval ──────────────────────────────────────────────────────── + def edges(self, concept: str, relation: str, *, reverse: bool = False, + limit: int = 6) -> list[dict[str, Any]]: + concept = concept.strip().lower() + column, other = ("end", "start") if reverse else ("start", "end") + rows = self.connection.execute( + f"SELECT rel, start, end, weight, surface_text, dataset FROM edge " + f"WHERE {column}=? AND rel=? ORDER BY weight DESC, start, end LIMIT ?", + (concept, relation, limit)).fetchall() + return [{"rel": r[0], "start": r[1], "end": r[2], "weight": r[3], + "surface_text": r[4], "dataset": r[5], + "provenance": f"conceptnet57:{r[0]}:{r[1]}->{r[2]}"} for r in rows] + + def neighborhood(self, concept: str, *, limit_per_relation: int = 4) -> dict[str, list[dict[str, Any]]]: + out: dict[str, list[dict[str, Any]]] = {} + for relation in RELATION_TEMPLATES: + if relation == "RelatedTo": + continue # too diffuse for a summary; available via edges() + found = self.edges(concept, relation, limit=limit_per_relation) + if found: + out[relation] = found + return out + + # ── rendering ──────────────────────────────────────────────────────── + @staticmethod + def render_edge(edge: dict[str, Any]) -> str: + surface = (edge.get("surface_text") or "").strip() + if surface: + text = _SURFACE_BRACKETS.sub("", surface).lstrip("*").strip() + if text: + return text[0].upper() + text[1:].rstrip(".") + "." + template = RELATION_TEMPLATES.get(edge["rel"], "{start} relates to {end}") + text = template.format(start=edge["start"], end=edge["end"]) + return text[0].upper() + text[1:] + "." + + def answer(self, question: str) -> dict[str, Any]: + """Route a commonsense question form; abstain explicitly otherwise.""" + text = " ".join(question.strip().split()) + for pattern, relations in QUESTION_ROUTES: + match = pattern.match(text) + if not match: + continue + concept = match.group(1).strip().lower() + collected: list[dict[str, Any]] = [] + for spec in relations: + relation, _, direction = spec.partition(":") + collected.extend(self.edges(concept, relation, reverse=(direction == "end"))) + if not collected: + return {"status": "withheld", "operation": "conceptnet_no_admitted_edges", + "concept": concept, + "response": f"The commonsense index has no admitted edges answering that about {concept}.", + "claims": []} + seen: set[str] = set() + lines = [] + claims = [] + for edge in collected: + rendered = self.render_edge(edge) + if rendered in seen: + continue + seen.add(rendered) + lines.append(rendered) + claims.append({"provenance": edge["provenance"], "weight": edge["weight"], + "dataset": edge["dataset"]}) + if len(lines) >= 5: + break + return {"status": "answered", "operation": "conceptnet_commonsense", + "concept": concept, "response": " ".join(lines), "claims": claims} + return {"status": "unrouted", "operation": "conceptnet_form_not_registered", + "response": "", "claims": []} diff --git a/root/wander-train/oracle_release_runtime_aa.py b/root/wander-train/oracle_release_runtime_aa.py new file mode 100644 index 0000000000000000000000000000000000000000..248f3375a9a24e37523f0e245cfe21cedba2a6e8 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_aa.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Runtime AA — evidence-mined tiered cues for the explanatory routes (additive over Z). + +M24's cue inventories were authored by guessing and the route stress test showed +the cost: how 10%, what_causes 28%, when 65% answer-rate. Iteration 4 mined +400,000 admitted sentences to find the forms the corpus ACTUALLY uses, and the +guesses had missed the most frequent ones outright: + + cause : "led to" 1,117 · "so that" 1,012 · "as a result" 372 · "result of" 340 + process : "uses" 3,647 · "when the" 3,580 · "is called" 3,108 · "is/are formed" 263 + date : "was released" 1,038 · "was built" 560 · "took place" 323 + ("built" was in the question regex but NOT in the inception cues at all) + +Widening naively would trade precision for recall, so the cues are TIERED: a +specific form is preferred, and a broad one is consulted only when no specific +form matches. Frequency alone does not promote a cue — "makes" is frequent but +weakly causal, so it sits in the last tier. + +Also fixes a sentence-quality defect the harness surfaced: sentence extraction +sometimes yields a fragment starting mid-parenthetical ("1938) is a psychologist +..."), which is cited and correct but reads as broken. Such fragments are +skipped in favour of the next admitted sentence.""" + +from __future__ import annotations + +import re +import time + +from oracle_release_runtime_z import OracleReleaseRuntimeZ + +# tier 1 = explicitly the relation; tier 2 = strongly indicative; tier 3 = weak fallback +CAUSE_TIERS = [ + re.compile(r"\b(caused by|is caused|are caused|due to|results? from|triggered by|produced by)\b", re.I), + re.compile(r"\b(because|led to|leads to|as a result|result of|gives rise to)\b", re.I), + re.compile(r"\b(causes|makes|so that|therefore|happens when|occurs when)\b", re.I), +] +PROCESS_TIERS = [ + re.compile(r"\b(happens when|occurs when|forms? when|is formed|are formed|works? by|by which|through which)\b", re.I), + re.compile(r"\b(process|is made|are made|begins when|starts when|when there is)\b", re.I), + re.compile(r"\b(uses|when the|is called|moves|produces)\b", re.I), +] +# Date cues are conditioned on the VERB ACTUALLY ASKED, not a fixed tier order. +# A flat specific-first ordering regressed "When did WWII start?" to "The Tanggu +# Truce ... was signed in 1933" because `was signed` outranked `began`. The asked +# verb is the correct selector: it says which event the questioner means. +VERB_CUES = { + "start": [r"\b(began|begun|started|broke out|was launched)\b", r"\b(was founded|was established|was created)\b"], + "begin": [r"\b(began|begun|started|broke out|was launched)\b", r"\b(was founded|was established|was created)\b"], + "end": [r"\b(ended|finished|collapsed|surrendered|was dissolved)\b"], + "found": [r"\b(was founded|was established|was created|founded)\b"], + "built": [r"\b(was built|was constructed|built between|was completed)\b"], + "build": [r"\b(was built|was constructed|built between|was completed)\b"], + "discover": [r"\b(was discovered|discovered)\b"], + "invent": [r"\b(was invented|invented)\b"], + "born": [r"\b(was born|born)\b"], + "die": [r"\b(died|was killed)\b"], + "happen": [r"\b(took place|happened|occurred|was held)\b"], + "occur": [r"\b(took place|happened|occurred|was held)\b"], +} +INCEPT_GENERAL = [ + re.compile(r"\b(began|begun|started|was founded|was built|was established|was created|took place|broke out)\b", re.I), + re.compile(r"\b(opened|ended|was released|was signed|was discovered|was invented|born|died)\b", re.I), +] +YEAR = re.compile(r"\b(1[0-9]{3}|20[0-2][0-9])\b") +FRAGMENT = re.compile(r"^\s*(?:[a-z]|\d{1,4}\s*[)\]])") # "1938) is ..." / lowercase start + + +class OracleReleaseRuntimeAA(OracleReleaseRuntimeZ): + def _pick_tiered(self, topic: str, tiers: list, *, require_year: bool = False) -> list[dict]: + bundle = self._bundle(topic, n=60) + if not bundle: + singular = topic[:-1] if topic.endswith("s") and not topic.endswith("ss") else None + if singular and len(singular) > 2: + bundle = self._bundle(singular, n=60) + if not bundle: + return [] + usable = [c for c in bundle if not FRAGMENT.match(str(c.get("sentence") or ""))] + if not usable: + usable = bundle + for tier in tiers: # specific first, broad only as fallback + hits = [] + for claim in usable: + sentence = str(claim.get("sentence") or "") + if require_year and not YEAR.search(sentence): + continue + if tier.search(sentence): + hits.append(claim) + if hits: + return hits + return [] + + def _explanatory_turn(self, text: str) -> dict | None: # override Y + from oracle_release_runtime_y import CAUSE_Q, HOW_Q, WHEN_Q + stripped = " ".join(text.strip().split()) + + match = CAUSE_Q.match(stripped) + if match: + topic = match.group("t").strip() + return self._render(topic, self._pick_tiered(topic, CAUSE_TIERS), + "cause", "capability:what_causes", f"what causes {topic}") + + match = HOW_Q.match(stripped) + if match: + topic = match.group("t").strip() + return self._render(topic, self._pick_tiered(topic, PROCESS_TIERS), + "process", "capability:how_does", f"how {topic} works") + + match = WHEN_Q.match(stripped) + if match: + topic = match.group("t").strip() + verb = (match.group("v") or "").lower() + tiers = [] + for stem, patterns in VERB_CUES.items(): + if verb.startswith(stem): + tiers = [re.compile(pat, re.I) for pat in patterns] + break + tiers = tiers + INCEPT_GENERAL + hits = self._pick_tiered(topic, tiers, require_year=True) \ + or self._pick_tiered(topic, [YEAR], require_year=True) + return self._render(topic, hits, "inception", "capability:when_did", + f"when {topic} {match.group('v')}") + return None + + def chat(self, text: str, session_id: str = "default") -> dict: + started = time.perf_counter() + result = None + try: + result = self._explanatory_turn(text) + except Exception: # noqa: BLE001 + result = None + if result is not None: + result["latency_ms"] = round(1000.0 * (time.perf_counter() - started), 1) + result["zero_model_gate"] = {"lm_calls": 0, "transformer_calls": 0} + return result + return super().chat(text, session_id) diff --git a/root/wander-train/oracle_release_runtime_ac.py b/root/wander-train/oracle_release_runtime_ac.py new file mode 100644 index 0000000000000000000000000000000000000000..18dc38f6e2a785b48debb608d65c2a0131ffb288 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_ac.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Runtime AC — explain-simply picks the identifying sentence (additive over AB). + +Iteration 6 diagnosis. `_definitional_score` returns 0 for every sentence on a +biographical page (it keys on copular definition patterns that proper-noun prose +does not use), so selection fell through to the tiebreak `-len(words)` — the +SHORTEST sentence wins. That systematically chose trivia and fragments over the +sentence that identifies the subject: + + "Explain Paco Gento simply." -> "He played for Spain national team." + "Explain Arthur C. Clarke simply." -> "He also wrote many short stories." + "Explain Bernard Montgomery simply."-> "Later, in World War II." (a fragment) + +In encyclopedic prose the FIRST sentence of a page is by convention the one that +says what the subject is. So on a score tie the tiebreak now prefers EARLIER +position rather than fewer words, and sentences that cannot stand alone (no +finite verb, or opening mid-parenthetical) are skipped. Brevity is still +preferred among sentences that actually define. + +Note on the metric that found this: `explain_simply` on-topic looked like the +same false-drift artifact that `list` produced, but it was not. For an +"explain X" question, mentioning X is not a proxy for relevance — it IS the +requirement, because an explanation that never identifies its subject has not +explained it.""" + +from __future__ import annotations + +import re +import time + +from oracle_release_runtime_ab import OracleReleaseRuntimeAB +from oracle_release_runtime_m import SIMPLY, _definitional_score +from stage5m13e8_relevance_gate_v1 import gate_bundle + +_WORD = chr(92) + "b" # backslash-b built at runtime; heredocs eat the literal +FINITE_VERB = re.compile(_WORD + r"(is|are|was|were|has|have|had|does|did|can|" + r"could|will|would|became|becomes|plays|played|wrote|" + r"made|makes|lives|lived|died|won|led|serves|served)" + _WORD, re.I) +OPENS_MID = re.compile(r"^\s*(?:[a-z]|\d{1,4}\s*[)\]]|later,|however,|also,|then,)", re.I) + + +def _standalone(sentence: str) -> bool: + """Can this sentence stand alone as an identification?""" + text = str(sentence or "").strip() + if len(text.split()) < 4 or OPENS_MID.match(text): + return False + return bool(FINITE_VERB.search(text)) + + +class OracleReleaseRuntimeAC(OracleReleaseRuntimeAB): + def _simply_turn(self, text: str, session_id: str) -> dict | None: # override M + match = SIMPLY.match(text.strip()) + if not match: + return None + topic = (match.group("t") or match.group("t2") or "").strip() + if not topic: + return None + bundle = self._bundle(topic, n=8) + admitted, _ = gate_bundle(topic, bundle) + gloss = self._gloss(topic) + if not admitted and not gloss: + return {"status": "withheld", "path": "capability:explain_simply", + "response": f"No provenance-admitted material on {topic} is available to explain.", + "provenance": {"kind": "simple_register", "topic": topic}} + parts: list[str] = [] + premises: list[str] = [] + if admitted: + order = {c["claim_id"]: i for i, c in enumerate(bundle)} + standalone = [c for c in admitted if _standalone(c.get("sentence"))] + pool = standalone or admitted + short_defs = [c for c in pool + if len(c["sentence"].split()) <= 30 + and _definitional_score(topic, c["sentence"]) >= 6] + if short_defs: + lead = max(short_defs, key=lambda c: (_definitional_score(topic, c["sentence"]), + -len(c["sentence"].split()))) + else: + # No scored definition (typical of proper-noun pages): take the + # page's own opening sentence, which is the identifying one. + lead = min(pool, key=lambda c: order.get(c["claim_id"], 10_000)) + parts.append(f"{lead['sentence']} [{lead['claim_id']}]") + premises.append(lead["claim_id"]) + if gloss: + parts.append(f"The dictionary sense: {gloss}. [wiktionary:{topic.lower()}:sense1]") + premises.append(f"wiktionary:{topic.lower()}") + return {"status": "answered", "path": "capability:explain_simply", + "response": "In simple terms: " + " ".join(parts), + "provenance": {"kind": "simple_register", "topic": topic, "premises": premises}} + + def _timeline_turn(self, text: str, session_id: str) -> dict | None: # override J + result = super()._timeline_turn(text, session_id) + if not result or result.get("status") != "answered": + return result + body = str(result.get("response", "")) + distinct = {y for y in year_tokens(body)} + if len(distinct) >= 2: + return result + topic_note = "the admitted material yields fewer than two distinct dates" + return {"status": "withheld", "path": "capability:timeline", + "response": f"A timeline needs at least two distinct dated statements; " + f"{topic_note}. I will not present a measurement as a date.", + "provenance": {"kind": "temporal_projection", + "result": "insufficient_distinct_dates"}} + + def chat(self, text: str, session_id: str = "default") -> dict: + started = time.perf_counter() + result = None + try: + result = self._simply_turn(" ".join(text.strip().split()), session_id) + except Exception: # noqa: BLE001 + result = None + if result is not None: + result["latency_ms"] = round(1000.0 * (time.perf_counter() - started), 1) + result["zero_model_gate"] = {"lm_calls": 0, "transformer_calls": 0} + return result + return super().chat(text, session_id) + + +# ── timeline: a year token is not the same thing as a number ──────────────── +# The route treated any 4-digit token as a date, so "One megasecond contains +# 1000 kiloseconds" was rendered as the year 1000, and a page with one date +# repeated produced a single-point "timeline". A date must not be a measurement, +# and a timeline needs at least two DISTINCT years. +UNIT_AFTER = re.compile( + r"^(?:kilo|mega|giga|milli|micro|nano)?" + r"(?:second|meter|metre|gram|litre|liter|byte|watt|hertz|mile|foot|feet|inch|" + r"year|month|week|day|hour|minute|people|person|km|kg|cm|mm|ft|lb)s?$", re.I) + + +def year_tokens(sentence: str) -> list[int]: + """4-digit years in a sentence, excluding numbers that are measurements.""" + text = str(sentence or "") + out: list[int] = [] + for match in re.finditer(r"(1[0-9]{3}|20[0-2][0-9])", text): + tail = text[match.end():].lstrip() + next_word = tail.split()[0].strip(".,;:)") if tail.split() else "" + if next_word and UNIT_AFTER.match(next_word): + continue # "1000 kiloseconds" is a quantity, not a date + out.append(int(match.group(1))) + return out diff --git a/root/wander-train/oracle_release_runtime_ag.py b/root/wander-train/oracle_release_runtime_ag.py new file mode 100644 index 0000000000000000000000000000000000000000..1e23335ff72ad103064ca00d4f6319916d0c23bf --- /dev/null +++ b/root/wander-train/oracle_release_runtime_ag.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Runtime AG - cite a genuine hypernym chain (additive over AF). + +AF answered correctly but CITED INCORRECTLY: + + "Yes. The lexical record places gray whale under whale: + organism -> living thing -> whole -> object -> physical entity -> gray whale" + +The claimed hypernym ("whale") does not appear in the chain shown. Corpus v1 +stored each synset's total path length as the depth of every node on that path, +so all hypernyms of a sense shared one meaningless value and ordering by it +produced a sequence that was not a hypernym path at all. + +For a system whose entire claim is shown provenance, citing a chain that does not +contain the cited term is the most serious defect available - worse than the +parser bug, because the answer looks right and the evidence is fabricated +structure. + +Corpus v2 stores each node's own distance from the root, so ascending depth is a +genuine general-to-specific ordering. This runtime reads v2, renders the chain in +that order, and guarantees the cited hypernym appears in what it shows. It also +never presents more than the real path: nodes are drawn from the lemma's own +closure only. + +No neural network; no LM call. +""" + +from __future__ import annotations + +import sqlite3 +import time +from pathlib import Path + +from oracle_release_runtime_af import OracleReleaseRuntimeAF + +WORDNET_CORPUS_V2 = (__import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/fable-content/oracle-m-series-2026-08-15" + "/m31_wordnet_corpus_v2.sqlite3") + + +class OracleReleaseRuntimeAG(OracleReleaseRuntimeAF): + def __init__(self, *args, wordnet_corpus: str = WORDNET_CORPUS_V2, **kwargs) -> None: + super().__init__(*args, wordnet_corpus=wordnet_corpus, **kwargs) + + def _chain_for(self, lemma: str, limit: int = 6) -> list: + """General -> specific, by each node's own distance from the root.""" + rows = self._wn.execute( + "SELECT hypernym FROM lemma_closure WHERE lemma=? ORDER BY depth ASC, hypernym ASC " + "LIMIT ?", (lemma, limit)).fetchall() + return [r[0] for r in rows] + + def _render_chain(self, lemma: str, hypernym: str | None) -> str: + """The shown chain must contain the cited hypernym, or it is not evidence.""" + chain = self._chain_for(lemma, limit=6) + if hypernym and hypernym not in chain: + row = self._wn.execute( + "SELECT depth FROM lemma_closure WHERE lemma=? AND hypernym=?", + (lemma, hypernym)).fetchone() + if row is not None: + chain = self._chain_for(lemma, limit=5) + if hypernym in chain: + chain.remove(hypernym) + chain = chain[:4] + [hypernym] + return " -> ".join(chain + [lemma]) if chain else lemma + + def _wordnet_turn(self, text: str) -> dict | None: + if self._wn is None: + return None + parsed = self._parse_kind(text) + if parsed is None: + return None + x, y = parsed + lemma = self._present(x) + if lemma is None: + return None + hit = self._closure_hit(lemma, y) + article_x = "an" if x[:1] in "aeiou" else "a" + article_y = "an" if y[:1] in "aeiou" else "a" + if hit is not None: + hypernym, _depth, sense = hit + shown = self._render_chain(lemma, hypernym) + return { + "status": "answered", + "path": "wordnet_kind_direct", + "response": (f"Yes. The lexical record places {lemma} under {hypernym}, " + f"from the general to the specific: {shown}. On that record " + f"{article_x} {x} is {article_y} {y}. [cited:wordnet:{sense}]"), + "provenance": {"kind": "lexical_corpus", "corpus": "wordnet", + "relation": "hypernym", "lemma": lemma, + "hypernym": hypernym, "sense": sense, + "chain": self._chain_for(lemma) + [lemma]}, + } + shown = self._render_chain(lemma, None) + return { + "status": "answered", + "path": "wordnet_kind_excluded", + "response": (f"No. The lexical record places {lemma} on a different line, " + f"from the general to the specific: {shown}. {y.capitalize()} does " + f"not appear anywhere in that chain, and the chain is complete, so " + f"this is an exclusion rather than a gap in the record. " + f"[cited:wordnet:{lemma}]"), + "provenance": {"kind": "lexical_corpus", "corpus": "wordnet", + "relation": "hypernym_excluded", "lemma": lemma, + "chain": self._chain_for(lemma) + [lemma]}, + } diff --git a/root/wander-train/oracle_release_runtime_ah.py b/root/wander-train/oracle_release_runtime_ah.py new file mode 100644 index 0000000000000000000000000000000000000000..5c8dc6bea945c0f125803a6d3f82ef8629f02ebd --- /dev/null +++ b/root/wander-train/oracle_release_runtime_ah.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Runtime AH - honest exclusion rendering (additive over AG). + +AG fixed the affirmative citation but left the SAME defect class in the exclusion +path. Two problems, both of which fabricate structure: + +1. Sense interleaving. The closure is a union over all senses, so rendering it as + one arrow-path splices unrelated lineages: + + "guinea pig: entity -> physical entity -> causal agent -> object -> + person -> whole -> guinea pig" + + That path does not exist. It mixes the rodent with the separate sense of a + person used as a test subject. + +2. Truncated scope. The text asserted a term "does not appear anywhere in that + chain" while displaying only a six-node prefix, inviting the reader to verify + an exhaustive claim against partial evidence. + +Fix: an exclusion no longer claims a path. It reports the most specific categories +the record actually assigns - which are the sense-characteristic ones - as a SET, +and states plainly that the exclusion was checked against the complete closure +while what is shown is a summary of it. The affirmative path still shows a real +chain, because there a single genuine lineage exists and contains the cited term. + +Say only what the record supports, and show evidence in the shape it actually has. +No neural network; no LM call. +""" + +from __future__ import annotations + +from oracle_release_runtime_ag import OracleReleaseRuntimeAG + +SHOW_CATEGORIES = 6 + + +class OracleReleaseRuntimeAH(OracleReleaseRuntimeAG): + def _specific_categories(self, lemma: str, limit: int = SHOW_CATEGORIES) -> list: + """Deepest hypernyms = the sense-characteristic ones. A set, not a path.""" + rows = self._wn.execute( + "SELECT hypernym FROM lemma_closure WHERE lemma=? ORDER BY depth DESC, hypernym ASC " + "LIMIT ?", (lemma, limit)).fetchall() + return [r[0] for r in rows] + + def _closure_size(self, lemma: str) -> int: + row = self._wn.execute( + "SELECT COUNT(*) FROM lemma_closure WHERE lemma=?", (lemma,)).fetchone() + return int(row[0]) if row else 0 + + def _wordnet_turn(self, text: str) -> dict | None: + result = super()._wordnet_turn(text) + if result is None or result.get("path") != "wordnet_kind_excluded": + return result + prov = result.get("provenance", {}) + lemma = prov.get("lemma") + parsed = self._parse_kind(text) + if not lemma or parsed is None: + return result + _x, y = parsed + cats = self._specific_categories(lemma) + total = self._closure_size(lemma) + listed = ", ".join(cats) if cats else "no categories on record" + result["response"] = ( + f"No. The record classifies {lemma} under {listed}" + f"{' and other broader categories' if total > len(cats) else ''}. " + f"{y.capitalize()} is not among the {total} categories on record for it, and " + f"that record is complete, so this is an exclusion rather than a gap. " + f"[cited:wordnet:{lemma}]") + result["provenance"] = { + "kind": "lexical_corpus", "corpus": "wordnet", + "relation": "hypernym_excluded", "lemma": lemma, + "categories_shown": cats, "categories_total": total, + "note": ("categories are a set drawn from all senses, not a single lineage; " + "the exclusion was checked against the complete closure"), + } + return result diff --git a/root/wander-train/oracle_release_runtime_aj.py b/root/wander-train/oracle_release_runtime_aj.py new file mode 100644 index 0000000000000000000000000000000000000000..6832308b0a61bc22ca7c553142d44c829f041336 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_aj.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Runtime AJ - lexicon-informed splitting for determiner-free questions (additive over AI). + +M38 passed eight columns and still mis-parsed this: + + "Is hand luggage luggage?" -> x='hand', y='luggage luggage' + -> "No. The record classifies HAND under forepaw, paw, animal foot..." + +With no determiner the strict pattern cannot anchor, and the loose pattern's non-greedy +subject takes the shortest split. This is the M31 defect resurfacing in the form the M31 +fix did not cover, and no panel saw it because every generated question carries +determiners ("Is a X a Y?"). The route's own domain includes determiner-free forms; +the test distribution did not. + +Fix: when no determiner is available to anchor the split, use the admitted record as the +lexicon. Enumerate every split point and choose the longest subject for which BOTH sides +are lemmas the corpus carries. Purely structural, no heuristic weighting - the record +already knows which strings are terms. + + "Is hand luggage luggage?" + split 1: x='hand' y='luggage luggage' -> y not a lemma, rejected + split 2: x='hand luggage' y='luggage' -> both lemmas, ACCEPTED + +Determiner-bearing forms still take the strict path unchanged. No neural network. +""" + +from __future__ import annotations + +import re + +from oracle_release_runtime_ai import OracleReleaseRuntimeAI +from oracle_release_runtime_af import KIND_Q_STRICT, _FILLER, _LEAD +from oracle_release_runtime_ae import _forms + +_BODY = re.compile( + r"^(?:is|are)\s+(?:(?:an?|the|some)\s+)?(?P[a-z][a-z '\-]+?)\s*[.?!]*$", re.I) +MAX_WORDS = 8 + + +class OracleReleaseRuntimeAJ(OracleReleaseRuntimeAI): + def _valid_pair(self, x: str, y: str): + if not x or not y or x == y or len(x) < 2 or len(y) < 2: + return None + if set(_forms(x)) & set(_forms(y)): + return None + return x, y + + def _parse_kind(self, text: str): + t = " ".join(text.strip().split()) + t = " ".join(_FILLER.sub("", _LEAD.sub("", t)).split()) + match = KIND_Q_STRICT.match(t) + if match: + return self._valid_pair(match.group("x").strip().lower(), + match.group("y").strip().lower()) + body_match = _BODY.match(t) + if not body_match: + return super()._parse_kind(text) + words = body_match.group("body").strip().lower().split() + if len(words) < 2 or len(words) > MAX_WORDS: + return super()._parse_kind(text) + chosen = None + for cut in range(1, len(words)): + x = " ".join(words[:cut]) + y = " ".join(words[cut:]) + pair = self._valid_pair(x, y) + if pair is None: + continue + if self._present(x) and self._present(y): + chosen = pair # keep iterating: prefer the LONGEST subject + if chosen is not None: + return chosen + return super()._parse_kind(text) diff --git a/root/wander-train/oracle_release_runtime_am.py b/root/wander-train/oracle_release_runtime_am.py new file mode 100644 index 0000000000000000000000000000000000000000..ae57a19a294bff8a8831430e9146b25701447d56 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_am.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Runtime AM - record dialogue state from the FINAL result (additive over AL). + +M46 passed and two defects survived it, both visible the moment turn-1 used a route other +than summarize: + + "Is a gray whale a whale?" -> answered by wordnet_kind_direct + replay: "came via the DOCUMENT_RUNTIME route ... typed refusal" + "What causes rain?" -> answered by capability:what_causes + replay: "Nothing has been answered in this session yet" + +Cause: dialogue state is written inside runtime E's chat, but answers are produced at many +layers of the chain. Routes resolved ABOVE E never write state at all; routes resolved +after E (the whole AD..AL tier) leave E's record of the INNER result in place. The ACTS +layer therefore reads a state describing a different answer than the user received. + +Fix: the top of the chain records what the user actually saw. State is written once, from +the final result, after every turn - overwriting whatever inner layers recorded. Acts are +conversions over the delivered answer, so the delivered answer is what must be stored. + +M46 could not see this because every turn-1 probe in it was "Summarize X" - a single route, +which happens to be one that flows through E. The test distribution must cover the route +inventory, not one member of it. + +No neural network; no LM call. +""" + +from __future__ import annotations + +import re + +from oracle_release_runtime_al import OracleReleaseRuntimeAL + +CITE = re.compile(r"\[[a-z0-9_]+:?[a-z0-9_.\-]*\]") +ACT_PATHS = ("act:provenance_replay", "act:acknowledge", "act:continue_topic", "act:refine") + + +class OracleReleaseRuntimeAM(OracleReleaseRuntimeAL): + def chat(self, text: str, session_id: str = "default") -> dict: + result = super().chat(text, session_id) + path = str(result.get("path", "")) + # acts are conversions OVER the last answer; they must not overwrite it + if path in ACT_PATHS: + return result + try: + slot = self.dialogue.setdefault(session_id, {}) + slot["last_result"] = { + "path": result.get("path"), + "provenance": result.get("provenance"), + "status": result.get("status"), + "citations": CITE.findall(str(result.get("response", ""))), + } + except Exception: # noqa: BLE001 + pass + return result diff --git a/root/wander-train/oracle_release_runtime_c.py b/root/wander-train/oracle_release_runtime_c.py new file mode 100644 index 0000000000000000000000000000000000000000..3656dcf768a5b6b3346de96a7301fa8454614022 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_c.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Front-door successor C (additive over B): essay route + definitional route. + +Essay route: "write an essay about X" -> the H2 gated document compiler over +the topic page plus up to two related pages (title matches carrying a +distinctive topic token, relevance-gated). Output is a structured document +whose thesis/conclusions/analyses/claims chain is verified by +recover_document before release; an unverifiable document is withheld. + +Definitional route (closes the registered debt): "what is a/an X?" answers +with the gated definitional lead of X's page when the page exists, instead of +sense clarification.""" + +from __future__ import annotations + +import re +import time + +from oracle_release_runtime_b import OracleReleaseRuntimeB +from stage5m13e8_document_compiler_v1 import compile_section, compile_document, recover_document +from stage5m13e8_paragraph_compiler_v1 import _definitional_score +from stage5m13e8_relevance_gate_v1 import topic_tokens, classify_claim + +ESSAY = re.compile(r"^(?:please )?(?:write|compose)(?: me)?(?: a| an)?(?: short| brief)?(?: \d+[- ]word)? essay (?:about|on) (?P.+?)\.?$", re.I) +DEFINITIONAL = re.compile(r"^what is (?:a |an |the )?(?P[\w \-']{2,60})\?$", re.I) + + +class OracleReleaseRuntimeC(OracleReleaseRuntimeB): + def _bundle(self, topic: str, n: int = 6) -> list[dict]: + page = self.bench_query.page_id(topic) + if not page: + return [] + return [{"claim_id": c, "page": topic, "sentence": s} + for c, s in self.bench_query.page_sentences(page)[:n]] + + def _related_titles(self, topic: str, cap: int = 2) -> list[str]: + distinctive, _ = topic_tokens(topic) + if not distinctive: + return [] + token = sorted(distinctive)[0] + # indexed prefix range on title_alias (no full scan over 6M articles) + rows = self.bench_query.connection.execute( + "SELECT a.title FROM title_alias t JOIN article a ON a.page_id = t.page_id " + "WHERE t.normalized_title >= ? AND t.normalized_title < ? LIMIT 12", + (token, token + "￿")).fetchall() + related = [] + for (title,) in rows: + if title.lower() == topic.lower(): + continue + if classify_claim(topic, title, title) in ("explanatory", "exact_topic"): + related.append(title) + if len(related) >= cap: + break + return related + + def _essay_turn(self, text: str) -> dict | None: + match = ESSAY.match(text.strip()) + if not match: + return None + topic = match.group("topic").strip() + topics = [topic] + self._related_titles(topic) + bundles = [(t, self._bundle(t), None) for t in topics] + bundles = [(t, b, None) for t, b, _ in bundles if b] + if not bundles: + return {"status": "withheld", "path": "gated_document", + "response": f"No provenance-admitted page for {topic} is available to ground an essay.", + "provenance": {"kind": "gated_document", "topic": topic}} + section = compile_section(f"The record on {topic}", bundles) + document = compile_document(f"A provenance-bounded account of {topic}", [section]) + if not recover_document(document): + return {"status": "withheld", "path": "gated_document", + "response": "Document failed its own derivation-chain verification and is withheld.", + "provenance": {"kind": "gated_document", "topic": topic, + "failure": "recover_document"}} + return {"status": "answered", "path": "gated_document", + "response": document["text"], + "provenance": {"kind": "gated_document", "topics": topics, + "exclusions": [{"page": e["page"], "relevance": e["relevance"]} + for e in section["exclusions"]]}} + + def _definitional_turn(self, text: str) -> dict | None: + match = DEFINITIONAL.match(text.strip()) + if not match: + return None + term = match.group("term").strip() + bundle = self._bundle(term.title()) or self._bundle(term) + if not bundle: + return None # fall through to sense machinery + lead = max(bundle, key=lambda c: _definitional_score(term, c["sentence"])) + if _definitional_score(term, lead["sentence"]) < 6: + return None + return {"status": "answered", "path": "definitional_encyclopedic", + "response": f"{lead['sentence']} [{lead['claim_id']}]", + "provenance": {"kind": "dbpedia_page", "page": lead["page"], + "claim_id": lead["claim_id"]}} + + def chat(self, text: str, session_id: str = "default") -> dict: + started = time.perf_counter() + normalized = " ".join(text.strip().split()) + result = None + for route in (lambda: self._essay_turn(normalized), + lambda: self._definitional_turn(normalized)): + try: + result = route() + except Exception: # noqa: BLE001 + result = None + if result is not None: + break + if result is None: + return super().chat(text, session_id) + result["latency_ms"] = round(1000.0 * (time.perf_counter() - started), 1) + result["zero_model_gate"] = {"lm_calls": 0, "transformer_calls": 0} + return result diff --git a/root/wander-train/oracle_release_runtime_e.py b/root/wander-train/oracle_release_runtime_e.py new file mode 100644 index 0000000000000000000000000000000000000000..330ac001b3a76b1fb87713cc7f29637cebea74ee --- /dev/null +++ b/root/wander-train/oracle_release_runtime_e.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Runtime E — conversation-act conversion layer, slice 1 (Ben's conversion +thesis: dialogue is a finite inventory of typed conversions, not an emergent +capability). + +Acts this slice: + provenance_replay "why did you say that / how do you know / source?" + -> re-present the last answer's provenance (addresses, + rule+premises, or the typed reason for a withhold) + continue_topic "tell me more / what else / more about that" + -> next admitted claims from the last topic's page, + composed and verified like any paragraph + acknowledge "thanks / ok / great" -> typed acknowledgment, no fetch + +Dialogue state is a per-session record {last_result, last_topic, last_offset} +— a dictionary, not a model. Every act is a conversion over existing +machinery; nothing generates.""" + +from __future__ import annotations + +import re +import time + +from oracle_release_runtime_d import OracleReleaseRuntimeD +from stage5m13e8_paragraph_compiler_v2 import compile_paragraph_v2, recover_program_v2 + +WHY = re.compile(r"^(?:why(?: did you say that| do you say that)?|how do you know(?: that)?|" + r"what(?:'s| is) your source(?: for that)?|source\??|prove it|citation\??)[.?!]?$", re.I) +MORE = re.compile(r"^(?:tell me more|more|what else|go on|continue|more about (?:that|it))[.?!]?$", re.I) +ACK = re.compile(r"^(?:thanks|thank you|ok|okay|got it|great|cool|nice|perfect)[.?!]?$", re.I) + + +class OracleReleaseRuntimeE(OracleReleaseRuntimeD): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.dialogue: dict[str, dict] = {} + + # ── act conversions ────────────────────────────────────────────────── + def _act_provenance_replay(self, state: dict) -> dict: + last = state.get("last_result") + if not last: + return {"status": "withheld", "path": "act:provenance_replay", + "response": "Nothing has been answered in this session yet, so there is no provenance to replay."} + prov = last.get("provenance") or {} + kind = prov.get("kind", "unknown") + lines = [f"My previous answer came via the {last.get('path','?')} route."] + if kind == "dbpedia_page": + lines.append(f"It is the definitional lead of the page '{prov.get('page')}', claim {prov.get('claim_id')}.") + elif kind == "gated_paragraph": + lines.append(f"It composed {prov.get('public_claims',0)} page-addressed claims" + + (f" and {prov['user_claims']} claims from your own documents" if prov.get('user_claims') else "") + + "; every sentence carries its claim address in brackets.") + if prov.get("excluded"): + lines.append("Excluded as insufficiently relevant (ledgered, not hidden): " + + ", ".join(f"{e['page']} ({e['relevance']})" for e in prov["excluded"][:4]) + ".") + elif kind == "conceptnet_edges": + claims = prov.get("claims", []) + lines.append(f"It rests on {len(claims)} ConceptNet edges: " + + "; ".join(c.get("provenance", "?") for c in claims[:4]) + ".") + elif kind == "session_world": + lines.append(f"It was computed from the {prov.get('statements',0)} statements you asserted " + "in this session's typed world — logic over your own facts, not retrieval.") + else: + lines.append("It was a typed refusal: no admitted evidence licensed an answer, so none was given.") + lines.append("Nothing in that answer was generated; each part is either addressed, derived, or withheld.") + return {"status": "answered", "path": "act:provenance_replay", "response": " ".join(lines), + "provenance": {"kind": "dialogue_state_replay"}} + + def _act_continue(self, state: dict, session_id: str) -> dict: + topic = state.get("last_topic") + if not topic: + return {"status": "withheld", "path": "act:continue_topic", + "response": "There is no active topic yet — ask about something first."} + offset = state.get("last_offset", 0) + 6 + page = self.anchors.resolve(topic) + if not page: + return {"status": "withheld", "path": "act:continue_topic", + "response": f"No further admitted material for {topic}."} + title = self.anchors.page_to_title.get(page, topic) + rows = self.bench_query.page_sentences(page)[offset:offset + 4] + if not rows: + return {"status": "withheld", "path": "act:continue_topic", + "response": f"The admitted record on {title} is exhausted — " + f"every stored sentence has been shown."} + claims = [{"claim_id": c, "page": title, "sentence": s} for c, s in rows] + paragraph, program = compile_paragraph_v2(topic, claims, evidence=3, aggregation_term=None) + if paragraph is None or not recover_program_v2(paragraph, program): + return {"status": "withheld", "path": "act:continue_topic", + "response": "Continuation failed verification and is withheld."} + state["last_offset"] = offset + return {"status": "answered", "path": "act:continue_topic", "response": paragraph, + "provenance": {"kind": "gated_paragraph", "topic": topic, "offset": offset}} + + # ── entry ──────────────────────────────────────────────────────────── + def chat(self, text: str, session_id: str = "default") -> dict: + started = time.perf_counter() + normalized = " ".join(text.strip().split()) + state = self.dialogue.setdefault(session_id, {}) + result = None + if WHY.match(normalized): + result = self._act_provenance_replay(state) + elif MORE.match(normalized): + result = self._act_continue(state, session_id) + elif ACK.match(normalized): + result = {"status": "answered", "path": "act:acknowledge", + "response": "Noted. The session state is kept; ask on.", + "provenance": {"kind": "dialogue_state"}} + if result is not None: + result["latency_ms"] = round(1000.0 * (time.perf_counter() - started), 1) + result["zero_model_gate"] = {"lm_calls": 0, "transformer_calls": 0} + else: + result = super().chat(text, session_id) + # topic tracking for continuation acts + path = result.get("path", "") + prov = result.get("provenance") or {} + if path == "anchored_summary": + match = re.match(r"^(?:tell me about|who is|who was|what do you know about) (?:the )?(.+?)[.?!]?$", + normalized, re.I) + if match: + state["last_topic"] = match.group(1).strip() + state["last_offset"] = 0 + elif path == "definitional_encyclopedic" and prov.get("page"): + state["last_topic"] = prov["page"] + state["last_offset"] = 0 + self.dialogue[session_id]["last_result"] = {k: result.get(k) for k in ("path", "provenance", "status")} + return result diff --git a/root/wander-train/oracle_release_runtime_h.py b/root/wander-train/oracle_release_runtime_h.py new file mode 100644 index 0000000000000000000000000000000000000000..8086d6e075486c81422dedc8a109663ccdb0489a --- /dev/null +++ b/root/wander-train/oracle_release_runtime_h.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Runtime H — capability cycle 3: list-N-things (additive over G).""" + +from __future__ import annotations + +import re +import time + +from oracle_release_runtime_g import OracleReleaseRuntimeG +from stage5m13e8_relevance_gate_v1 import gate_bundle + +NUMERALS = {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, + "seven": 7, "eight": 8, "nine": 9, "ten": 10, "a": 3, "an": 3, + "some": 3, "few": 3, "several": 4, "couple": 2} +LISTN = re.compile( + r"^(?:please |can you )?(?:list|give me|tell me|name|share)" + r"(?: (?P\d+|one|two|three|four|five|six|seven|eight|nine|ten|a few|some|several|a couple(?: of)?))?" + r" (?:facts?|things?|details?)(?: about| on| regarding)" + r"(?: the| my document| my)? (?P.+?)[.?!]?$" + r"|^what are (?:some|a few) facts? about(?: the| my document| my)? (?P.+?)[.?!]?$", + re.I) + + +class OracleReleaseRuntimeH(OracleReleaseRuntimeG): + def _listn_turn(self, text: str, session_id: str) -> dict | None: + match = LISTN.match(text.strip()) + if not match: + return None + groups = match.groupdict() + topic = (groups.get("t") or groups.get("t2") or "").strip() + if not topic: + return None + raw_n = (groups.get("n") or "some").lower().replace("a couple of", "couple").replace("a couple", "couple").replace("a few", "few") + n = int(raw_n) if raw_n.isdigit() else NUMERALS.get(raw_n, 3) + n = max(1, min(n, 10)) + user_claims = self.user_corpus.bundle(session_id, topic) + admitted, _ = gate_bundle(topic, self._bundle(topic, n=10)) + claims = (user_claims + admitted)[:n] + if not claims: + return {"status": "withheld", "path": "capability:list_n", + "response": f"No provenance-admitted facts about {topic} are available to list.", + "provenance": {"kind": "gated_list", "topic": topic}} + lines = [f"{i+1}. {c['sentence']} [{c['claim_id']}]" for i, c in enumerate(claims)] + shortfall = "" + if len(claims) < n: + shortfall = (f" Only {len(claims)} admitted fact" + f"{'s are' if len(claims) != 1 else ' is'} available for {topic}.") + return {"status": "answered", "path": "capability:list_n", + "response": " ".join(lines) + shortfall, + "provenance": {"kind": "gated_list", "topic": topic, + "requested": n, "delivered": len(claims), + "user_claims": len([c for c in claims if c["claim_id"].startswith("user:")])}} + + def chat(self, text: str, session_id: str = "default") -> dict: + started = time.perf_counter() + normalized = " ".join(text.strip().split()) + result = None + try: + result = self._listn_turn(normalized, session_id) + except Exception: # noqa: BLE001 + result = None + if result is not None: + result["latency_ms"] = round(1000.0 * (time.perf_counter() - started), 1) + result["zero_model_gate"] = {"lm_calls": 0, "transformer_calls": 0} + state = self.dialogue.setdefault(session_id, {}) + state["last_result"] = {k: result.get(k) for k in ("path", "provenance", "status")} + state["last_topic"] = result["provenance"].get("topic") + state["last_offset"] = 0 + return result + return super().chat(text, session_id) diff --git a/root/wander-train/oracle_release_runtime_k.py b/root/wander-train/oracle_release_runtime_k.py new file mode 100644 index 0000000000000000000000000000000000000000..519a1fe9d6ee7cac038bc6f03ca12279a79cfa19 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_k.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Runtime K — capability cycle 6: explain-why-different (additive over J).""" + +from __future__ import annotations + +import re +import time + +from oracle_release_runtime_j import OracleReleaseRuntimeJ +from stage5m13e8_paragraph_compiler_v1 import _definitional_score + +WHYDIFF = re.compile( + r"^(?:please )?(?:explain )?why are (?:the |a |an )?(?P.+?) and (?:the |a |an |my document )?(?P.+?) different[.?!]?$" + r"|^(?:please )?(?:explain )?why is (?:the |a |an )?(?P.+?) different from (?:the |a |an |my document )?(?P.+?)[.?!]?$" + r"|^(?:please )?(?:explain )?why (?:the |a |an )?(?P.+?) and (?:the |a |an |my document )?(?P.+?) are different[.?!]?$" + r"|^what makes (?:the |a |an )?(?P.+?) different from (?:the |a |an |my document )?(?P.+?)[.?!]?$", + re.I) +KIND = re.compile(r"\bis (?:a|an|the) ([\w -]{3,40}?)(?:[,.]| that| which| in| of| with)") + + +class OracleReleaseRuntimeK(OracleReleaseRuntimeJ): + def _whydiff_turn(self, text: str, session_id: str) -> dict | None: + match = WHYDIFF.match(text.strip()) + if not match: + return None + groups = match.groupdict() + x = next((groups[k] for k in ("x", "x2", "x3", "x4") if groups.get(k)), "").strip() + y = next((groups[k] for k in ("y", "y2", "y3", "y4") if groups.get(k)), "").strip() + if not x or not y: + return None + side_x, _ = self._side(x, session_id) + side_y, _ = self._side(y, session_id) + missing = [t for t, s in ((x, side_x), (y, side_y)) if not s] + if missing: + return {"status": "withheld", "path": "capability:why_different", + "response": f"I cannot explain the difference: no provenance-admitted material " + f"exists for {' or '.join(missing)}.", + "provenance": {"kind": "gated_whydiff", "missing": missing}} + lead_x = max(side_x, key=lambda c: _definitional_score(x, c["sentence"])) + lead_y = max(side_y, key=lambda c: _definitional_score(y, c["sentence"])) + kind_x_match = KIND.search(lead_x["sentence"]) + kind_y_match = KIND.search(lead_y["sentence"]) + kind_x = kind_x_match.group(1).strip() if kind_x_match else None + kind_y = kind_y_match.group(1).strip() if kind_y_match else None + if kind_x and kind_y and kind_x.lower() != kind_y.lower(): + reason = (f"Derived reason: they belong to different categories — {x} is attested as " + f"a {kind_x}, whereas {y} is attested as a {kind_y}.") + else: + reason = (f"Derived reason: the admitted records place them in " + f"{'the same attested category (' + kind_x + '); their difference lies in the particulars below' if kind_x else 'categories the leads do not name explicitly; the difference rests on the particulars below'}.") + response = (f"According to {lead_x['page']}, {lead_x['sentence']} [{lead_x['claim_id']}] " + f"According to {lead_y['page']}, {lead_y['sentence']} [{lead_y['claim_id']}] " + f"{reason} [derived:category_contrast<-{lead_x['claim_id'][:12]},{lead_y['claim_id'][:12]}]") + return {"status": "answered", "path": "capability:why_different", "response": response, + "provenance": {"kind": "gated_whydiff", "x": x, "y": y, + "derived": {"rule": "category_contrast", + "premises": [lead_x["claim_id"], lead_y["claim_id"]]}}} + + def chat(self, text: str, session_id: str = "default") -> dict: + started = time.perf_counter() + normalized = " ".join(text.strip().split()) + result = None + try: + result = self._whydiff_turn(normalized, session_id) + except Exception: # noqa: BLE001 + result = None + if result is not None: + result["latency_ms"] = round(1000.0 * (time.perf_counter() - started), 1) + result["zero_model_gate"] = {"lm_calls": 0, "transformer_calls": 0} + state = self.dialogue.setdefault(session_id, {}) + state["last_result"] = {k: result.get(k) for k in ("path", "provenance", "status")} + return result + return super().chat(text, session_id) diff --git a/root/wander-train/oracle_release_runtime_n.py b/root/wander-train/oracle_release_runtime_n.py new file mode 100644 index 0000000000000000000000000000000000000000..b2c44860a182983cd52411732a8bad097c0103f7 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_n.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Runtime N — runtime M with the K8 batch-1 grammar wired into the session +kernel (additive; the IR pipeline and all capability routes now parse through +the ~190-construction inventory).""" + +from __future__ import annotations + +from oracle_release_runtime_m import OracleReleaseRuntimeM +from stage5m13e2_construction_kernel_k8 import ConstructionKernelK8 +from stage5m13e4_migration2 import IRPipeline2 + + +class OracleReleaseRuntimeN(OracleReleaseRuntimeM): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.kernel = ConstructionKernelK8(compositional=True) + self.pipeline = IRPipeline2(self.kernel) diff --git a/root/wander-train/oracle_release_runtime_p.py b/root/wander-train/oracle_release_runtime_p.py new file mode 100644 index 0000000000000000000000000000000000000000..6701fc3e3776c745fc572eaa9d919541bc87f739 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_p.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Runtime P — M17a shadow arbitration (additive over O). + +Every chat turn is scored by the M-series release arbiter (frozen K138 bid +geometry + M-series families) and the ArbitrationRecord summary is attached to +the result as advisory metadata. Dispatch is UNCHANGED — the regex chain still +commits every action; the arbiter observes. Agreement per turn is recorded so +the live residue log accumulates the shadow scoreboard for the M17b promotion +gate. Advisory failure can never break a turn.""" + +from __future__ import annotations + +from oracle_release_runtime_o import OracleReleaseRuntimeO +from stage5m17_release_arbiter_v1 import arbitrate, path_family + + +class OracleReleaseRuntimeP(OracleReleaseRuntimeO): + def chat(self, text: str, session_id: str = "default") -> dict: + result = super().chat(text, session_id) + try: + record = arbitrate(" ".join(text.strip().split())) + regex_family = path_family(result.get("path")) + result["arbitration"] = { + "winner": record.winner, + "runner_up": record.runner_up, + "margin": round(record.margin, 3), + "floor": record.clarification_floor_applied, + "regex_family": regex_family, + "agrees": record.winner == regex_family if regex_family else None, + "mode": "shadow", + } + except Exception: # noqa: BLE001 — advisory only, never break dispatch + pass + return result diff --git a/root/wander-train/oracle_release_runtime_q.py b/root/wander-train/oracle_release_runtime_q.py new file mode 100644 index 0000000000000000000000000000000000000000..ed86e0c67028b6c01ebc504da038acf73ab701c5 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_q.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Runtime Q — M17b: arbiter AUTHORITATIVE at the fallback boundary (additive over P). + +Capture condition: the regex chain landed in the fallback tier +(document_runtime / session_relational) WITHOUT answering — i.e. it did not +understand the surface — AND the shadow arbiter bids a re-dispatchable family +with a clean margin (no clarification floor). Then, and only then, runtime Q +re-dispatches through the EXISTING route via a canonical surface: + + capture point 1 (capabilities + definitional): arbiter family + anchor-store + topic extraction -> canonical surface ("Give me a rundown on the Moon." + -> summarize + Moon -> "Summarize Moon.") -> parent chain. + capture point 2 (conversation acts): fixed canonical act surfaces + ("Keep going." -> "Tell me more."). + +Canonical-surface carriers mirror the K131 carrier stage deliberately; +typed-goal carrier removal is a later arc (K152 pattern). If re-dispatch fails +to answer, the ORIGINAL fallback result is returned unchanged — authority can +narrow to improvement-only, never degrade. Per the K147 law, no raw input is +ever placed into any response text by this layer (topics are anchor-store +titles, not user substrings). Each capture point has its own ablation flag.""" + +from __future__ import annotations + +import re + +from oracle_release_runtime_p import OracleReleaseRuntimeP +from stage5m17_release_arbiter_v1 import features as arb_features + +FALLBACK_PATHS = {"document_runtime", "session_relational"} +FALLBACK_OK = {"answered", "recorded"} + +# words never usable as topic candidates (marker vocabulary + function words) +TOPIC_BLOCK = { + "give", "me", "a", "an", "the", "of", "on", "about", "and", "or", "to", + "for", "in", "what", "whats", "what's", "how", "why", "is", "are", "do", + "does", "did", "you", "your", "i", "i'd", "id", "could", "would", "please", + "rundown", "overview", "summary", "summarize", "summarise", "recap", + "version", "short", "brief", "quick", "sum", "up", "like", "so", + "difference", "differences", "differ", "different", "between", "compare", + "contrast", "come", "timeline", "chronology", "history", "walk", "through", + "year", "by", "show", "list", "name", "few", "couple", "several", "some", + "things", "facts", "points", "items", "know", "tell", "talk", "mean", + "means", "actually", "explain", "break", "down", "plain", "language", + "simple", "simply", "simpler", "terms", "child", "follow", "it", "that", + "this", "one", "two", "three", "four", "five", "six", "seven", + "say", "said", "told", "know", "knew", "claim", "think", "thought", + "call", "called", "gist", "story", "handful", "nutshell", +} + +ACT_CANONICAL = { + "act_refine": "Simpler.", + "act_continue": "Tell me more.", + "act_provenance": "Why did you say that?", + "act_acknowledge": "Thanks!", +} + +NUMBER_WORDS = {"two": "two", "three": "three", "four": "four", "five": "five", + "six": "six", "seven": "seven", "2": "two", "3": "three", + "4": "four", "5": "five", "6": "six", "7": "seven"} + + +class OracleReleaseRuntimeQ(OracleReleaseRuntimeP): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.capture_capability_enabled = True # ablation flag, capture point 1 + self.capture_act_enabled = True # ablation flag, capture point 2 + + # ---------------------------------------------------- topic extraction + def _arb_topics(self, text: str) -> list[str]: + """Anchor-store titles found in the surface, longest-span first, + returned in positional order, marker vocabulary excluded.""" + words = [w.strip("'-") for w in re.findall(r"[A-Za-z][\w'-]*|\d+", text)] + words = [w for w in words if w] + titles = self.anchors.title_to_page + found: list[tuple[int, int, str]] = [] # (start, length, title) + taken: set[int] = set() + for n in (3, 2, 1): + for i in range(len(words) - n + 1): + if any(j in taken for j in range(i, i + n)): + continue + span = words[i:i + n] + if all(w.lower() in TOPIC_BLOCK for w in span): + continue + cand = " ".join(span) + if cand.lower() in titles: # anchor keys are case-normalized + found.append((i, n, cand)) + taken.update(range(i, i + n)) + return [t for _, _, t in sorted(found)] + + def _canonical_surface(self, family: str, text: str) -> str | None: + if family in ACT_CANONICAL: + return ACT_CANONICAL[family] + topics = self._arb_topics(text) + if family in ("compare", "why_different"): + if len(topics) < 2: + return None + a, b = topics[0], topics[1] + return (f"Compare {a} and {b}." if family == "compare" + else f"Why are {a} and {b} different?") + if not topics: + return None + topic = topics[0] + if family == "summarize": + return f"Summarize {topic}." + if family == "timeline": + return f"Give me a timeline of {topic}." + if family == "explain_simply": + return f"Explain {topic} simply." + if family == "list_n": + tokens = [w.lower() for w in re.findall(r"[A-Za-z]+|\d+", text)] + count = next((NUMBER_WORDS[t] for t in tokens if t in NUMBER_WORDS), "three") + return f"List {count} facts about {topic}." + if family == "definitional": + # the definiendum is the LAST content NP ("what would you say a + # glacier is?"), not the first + return f"What is a {topics[-1].lower()}?" + return None + + # ------------------------------------------------------------ dispatch + def chat(self, text: str, session_id: str = "default") -> dict: + result = super().chat(text, session_id) # P: regex dispatch + shadow record + arb = result.get("arbitration") + if (arb is None or arb.get("floor") + or result.get("path") not in FALLBACK_PATHS + or result.get("status") in FALLBACK_OK): + return result + family = arb.get("winner") + is_act = family in ACT_CANONICAL + if is_act and not self.capture_act_enabled: + return result + if not is_act and not self.capture_capability_enabled: + return result + canonical = self._canonical_surface(family, text) + if canonical is None: + return result + redispatch = super().chat(canonical, session_id) + # improvement-only authority: a family-correct honest withhold still + # beats the generic fallback clarification; anything else keeps the + # original result unchanged + if redispatch.get("status") not in ("answered", "withheld"): + return result + redispatch["path"] = f"arb:{family}->" + str(redispatch.get("path")) + redispatch["arbitration"] = dict(arb) + redispatch["arbitration"]["mode"] = "authoritative" + redispatch["arbitration"]["canonical_surface"] = canonical + redispatch["arbitration"]["capture_point"] = "act" if is_act else "capability" + return redispatch diff --git a/root/wander-train/oracle_release_runtime_s.py b/root/wander-train/oracle_release_runtime_s.py new file mode 100644 index 0000000000000000000000000000000000000000..a688817ac615b29337ed278be82f663f47843d1d --- /dev/null +++ b/root/wander-train/oracle_release_runtime_s.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Runtime S — M17b3: runtime R's capture logic driven by arbiter v3. +New file; P/Q/R untouched. Same fallback-boundary authority, improvement-only +rule, ablation flags, and canonical-surface carriers as R.""" + +from __future__ import annotations + +from oracle_release_runtime_o import OracleReleaseRuntimeO +from oracle_release_runtime_q import ACT_CANONICAL, FALLBACK_OK, FALLBACK_PATHS +from oracle_release_runtime_r import OracleReleaseRuntimeR +from stage5m17_release_arbiter_v3 import arbitrate as arbitrate_v3 +from stage5m17_release_arbiter_v3 import path_family + + +class OracleReleaseRuntimeS(OracleReleaseRuntimeR): + def chat(self, text: str, session_id: str = "default") -> dict: + result = OracleReleaseRuntimeO.chat(self, text, session_id) + try: + record = arbitrate_v3(" ".join(text.strip().split())) + except Exception: # noqa: BLE001 + return result + regex_family = path_family(result.get("path")) + arb = {"winner": record.winner, "runner_up": record.runner_up, + "margin": round(record.margin, 3), + "floor": record.clarification_floor_applied, + "regex_family": regex_family, + "agrees": record.winner == regex_family if regex_family else None, + "mode": "shadow", "arbiter": "v3"} + result["arbitration"] = arb + if (arb["floor"] or result.get("path") not in FALLBACK_PATHS + or result.get("status") in FALLBACK_OK): + return result + family = record.winner + is_act = family in ACT_CANONICAL + if is_act and not self.capture_act_enabled: + return result + if not is_act and not self.capture_capability_enabled: + return result + canonical = self._canonical_surface(family, text) + if canonical is None: + return result + redispatch = OracleReleaseRuntimeO.chat(self, canonical, session_id) + if redispatch.get("status") not in ("answered", "withheld"): + return result + redispatch["path"] = f"arb:{family}->" + str(redispatch.get("path")) + redispatch["arbitration"] = dict(arb) + redispatch["arbitration"]["mode"] = "authoritative" + redispatch["arbitration"]["canonical_surface"] = canonical + redispatch["arbitration"]["capture_point"] = "act" if is_act else "capability" + return redispatch diff --git a/root/wander-train/oracle_release_runtime_u.py b/root/wander-train/oracle_release_runtime_u.py new file mode 100644 index 0000000000000000000000000000000000000000..a534d045fe078984c48786a8559eaedca55aefca --- /dev/null +++ b/root/wander-train/oracle_release_runtime_u.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Runtime U — graceful-clarify terminal (additive over T). + +The last routing step for conversational smoothness: when a turn falls all the +way through the chain into the fallback tier WITHOUT being answered (the +document tier's "partial" evidence roundups and Wikipedia-address asks were +the wrong register for conversation — Ben-reported class, 2026-08-19), the +turn terminates in a typed clarify act instead: a short capability statement +in the machine's own voice. Answered / recorded / withheld results pass +through untouched; the arbitration record is preserved on the clarify so the +residue scoreboard keeps its labels. Per the K147 law, no user text rides in +the clarify.""" + +from __future__ import annotations + +import time + +from oracle_release_runtime_t import OracleReleaseRuntimeT + +CLARIFY_RESPONSE = ( + "I didn't fully parse that, so I won't guess. I can summarize a topic, " + "compare two things, list facts, give a timeline, explain something " + "simply, define a term, reason about why and what-if, and discuss any " + "document you teach me. A shorter, more direct phrasing usually lands." +) + +PASS_THROUGH = {"answered", "recorded", "withheld"} +FALLBACK_TIER = {"document_runtime", "session_relational"} + + +class OracleReleaseRuntimeU(OracleReleaseRuntimeT): + def chat(self, text: str, session_id: str = "default") -> dict: + started = time.perf_counter() + result = super().chat(text, session_id) + if (result.get("path") in FALLBACK_TIER + and result.get("status") not in PASS_THROUGH): + return {"status": "clarification", "path": "act:clarify", + "response": CLARIFY_RESPONSE, + "provenance": {"kind": "dialogue_state", + "note": "graceful clarify terminal"}, + "arbitration": result.get("arbitration"), + "latency_ms": round(1000.0 * (time.perf_counter() - started), 1), + "zero_model_gate": {"lm_calls": 0, "transformer_calls": 0}} + return result diff --git a/root/wander-train/oracle_release_runtime_w58.py b/root/wander-train/oracle_release_runtime_w58.py new file mode 100644 index 0000000000000000000000000000000000000000..bedde0373b0f0b681ff4b844e6f5643b846db59e --- /dev/null +++ b/root/wander-train/oracle_release_runtime_w58.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""oracle_release_runtime_w58 - JUSTIFICATION: why did you do that, answered from +the record rather than from a story about the record. + +Found by the salience pass over the held-out residue, not by me guessing. Among +the turns the machine could not answer was "Why did you choose that language?", +falling to act:clarify. It had just written a python program, recorded that it +wrote a python program, recorded which fold wrote it and what the execution +printed, and then could not say why python. + +That gap matters more than its size. The entire claim of this architecture is +that every answer carries a warrant which can be inspected. If the warrant can +only be inspected by reading a JSON file on a server, it is not a conversational +property, it is a filing system. A machine that can be asked "how did you get +that" and answers from its own witness record is doing the thing the design is +for. + +So this fold answers three questions about the PREVIOUS turn, and only from what +the record actually holds: + + why did you choose that the choice, and what determined it + how did you get that the warrant: derivation, computation, execution + what did you just do the operator and the construction it built + +The discipline, which is the whole point: it reports the reason ONLY when the +record supports it. If you did not name a language, the record shows the request +with no language in it and the output in python, so "you did not name one and +python is my default" is a reading of the record. If you did name one, the reason +is that you asked. And when the record holds no reason at all, it says exactly +that, because the failure mode here is not silence, it is a machine that +confabulates a plausible motive for its own behaviour. That is the one thing this +fold must never do. + +It also declines to speak when there is no previous turn, and hands back anything +that is not about the previous turn, so it cannot capture a conversation the way +the deduction fold did before w56. + +Hand-authored 2026-08-28. Judged by the gate. +""" + +from __future__ import annotations + +import re + +from oracle_release_runtime_w57 import OracleReleaseRuntimeW57 + +_WHY_CHOICE = re.compile( + r'\bwhy\s+(?:did|do)\s+you\s+(?:choose|pick|use|write|go with)\b' + r'|\bwhy\s+that\s+(?:language|one|approach|way)\b' + r'|\bwhat\s+made\s+you\s+(?:choose|pick|use)\b', re.I) +_HOW_GOT = re.compile( + r'\bhow\s+(?:did|do)\s+you\s+(?:get|know|work(?:ed)?\s+that\s+out|arrive|compute|' + r'calculate|check)\b|\bhow\s+was\s+that\s+(?:checked|verified)\b' + r'|\bshow\s+(?:me\s+)?(?:your|the)\s+(?:work|steps|working)\b', re.I) +_WHAT_DID = re.compile( + r'\bwhat\s+did\s+you\s+just\s+do\b|\bwhat\s+(?:route|operator)\s+was\s+that\b' + r'|\bhow\s+did\s+you\s+answer\s+that\b', re.I) + +_LANGS = ('python', 'javascript', 'go', 'bash', 'sql', 'c', 'rust', 'java', + 'ruby', 'typescript') +_WARRANT_NAME = { + 'test_execution': 'I ran it on this machine and compared the output', + 'deduction': 'I derived it, and each step names the rule that licensed it', + 'algebra': 'I solved it and then checked the answer by substituting it back', + 'computation': 'I computed it and kept the trace', + 'citation': 'I cited the source it came from', + 'differential_execution': 'I computed the expected result a second way and ' + 'only answered when the two agreed', +} + + +class OracleReleaseRuntimeW58(OracleReleaseRuntimeW57): + """Answers about the previous turn, grounded in that turn's witness record.""" + + def _prior_turn(self, session_id): + """The last recorded turn that was not itself one of these questions.""" + hist = getattr(self, '_paths', None) or {} + q = hist.get(session_id) + if not q: + return None + for entry in reversed(list(q)): + if str(entry.get('operator', '')).startswith('justify:'): + continue + return entry + return None + + @staticmethod + def _choice_reason(prior): + """Why this output, IF the record shows it. Otherwise None.""" + conv = prior.get('conversion') or {} + made = str(conv.get('to', '')) + asked = str(conv.get('from', '') or prior.get('in_surface', '')).lower() + lang = next((l for l in _LANGS if made.lower().startswith(l + '/') + or ('/' + l) in made.lower()), None) + if not lang: + return None + named = next((l for l in _LANGS if re.search(r'\b%s\b' % l, asked)), None) + if named == lang: + return ('I used %s because you asked for %s. The record of that turn ' + 'shows your request and the program it produced.' % (lang, named)) + if named is None: + return ('I used %s because your request did not name a language, and ' + '%s is what my code fold writes when none is named. That is a ' + 'default, not a judgement that %s was best for the task.' + % (lang, lang, lang)) + return ('I used %s although your request mentioned %s. My record shows the ' + 'mismatch and does not record a reason for it, so I am telling you ' + 'the mismatch rather than explaining it away.' % (lang, named)) + + def _justify(self, kind, prior): + conv = prior.get('conversion') or {} + vw = prior.get('verified_warrant') or {} + op = str(prior.get('operator', '') or 'an unrecorded route') + by = str(conv.get('produced_by', '') or 'an unrecorded fold') + + if kind == 'why_choice': + reason = self._choice_reason(prior) + if reason: + return reason + ' [self:witness]' + return ('My record for that turn shows the route (%s) and what it built ' + '(%s), but it does not record a reason for the choice you are ' + 'asking about. I am not going to invent one. [self:witness]' + % (op, conv.get('to') or 'no construction recorded')) + + if kind == 'how_got': + if not vw.get('kind'): + # Citation is a warrant too, and it lives in a different field. The + # first version of this fold read only verified_warrant and told a + # visitor that a properly sourced encyclopedic answer "should not + # have been presented as settled", which is a machine libelling its + # own good work. Check every warrant the record can carry. + cited = list((prior.get('relation') or {}).get('grounded_cited') or []) + if cited or prior.get('citations'): + return ('I did not compute that one, I quoted it. The record for ' + 'that turn carries %d citation%s%s, so the warrant is the ' + 'source rather than a derivation I can replay. ' + '[self:witness]' + % (prior.get('citations') or len(cited), + '' if (prior.get('citations') or len(cited)) == 1 else 's', + ' covering %s' % ', '.join(str(c) for c in cited[:4]) + if cited else '')) + return ('That answer came through %s, and my record for it holds no ' + 'warrant of any kind, neither a citation nor a check I can ' + 'replay. I should not have presented it as settled. ' + '[self:witness]' % op) + head = _WARRANT_NAME.get(str(vw['kind']), + 'the warrant recorded is of kind %s' % vw['kind']) + steps = [str(s) for s in (vw.get('steps') or [])] + if steps: + return ('%s. The recorded steps were: %s. [self:witness]' + % (head, '; then '.join(steps[:6]))) + return '%s, though the record kept no individual steps. [self:witness]' % head + + return ('That turn went through %s, produced by %s. It built a %s, from "%s" ' + 'to "%s"%s. [self:witness]' + % (op, by, conv.get('construction') or 'result', + str(conv.get('from', ''))[:60], + str(conv.get('to', ''))[:60], + ', warranted by %s' % vw['kind'] if vw.get('kind') else + ', with no warrant recorded')) + + def chat(self, text: str, session_id: str = 'default') -> dict: + s = ' '.join(str(text).strip().split()) + if _WHY_CHOICE.search(s): + kind = 'why_choice' + elif _HOW_GOT.search(s): + kind = 'how_got' + elif _WHAT_DID.search(s): + kind = 'what_did' + else: + return super().chat(s, session_id) + + prior = self._prior_turn(session_id) + if prior is None: + return {'status': 'withheld', 'path': 'justify:no_prior_turn', + 'response': ('You are asking about a previous answer and this is ' + 'the first turn I have recorded in this session, so ' + 'there is nothing for me to read.'), + 'provenance': {'kind': 'justification', 'prior_turn': None}, + 'latency_ms': 1} + + body = self._justify(kind, prior) + from collections import deque + hist = getattr(self, '_paths', None) + if hist is None: + hist = self._paths = {} + q = hist.get(session_id) + if q is None: + q = hist[session_id] = deque(maxlen=64) + prev = q[-1].get('turn', 0) if len(q) else 0 + q.append({'turn': prev + 1, 'in_surface': s[:120], + 'in_form': ['question_about_prior_turn'], + 'operator': 'justify:' + kind, + 'out_status': 'answered', 'in_terms': [], 'out_terms': [], + 'relation': {'grounded': [], 'grounded_cited': [], + 'introduced': [], 'unaccounted': []}, + 'carried_from_prior': ['witness_record'], + 'continuity_lexical': True, 'continuity': True, 'citations': 1, + 'residual': None, 'cost_ms': 1.0, + 'conversion': {'construction': 'ACCOUNT_OF_PRIOR_TURN', + 'from': s[:60], + 'to': str(prior.get('operator', ''))[:60], + 'produced_by': 'justify_w58'}, + 'verified_warrant': {'kind': 'self_citation', + 'steps': ['read the witness record of turn %s' + % prior.get('turn')]}}) + return {'status': 'answered', 'path': 'justify:' + kind, + 'response': body, + 'provenance': {'kind': 'justification', 'read_turn': prior.get('turn'), + 'prior_operator': prior.get('operator'), + 'source': 'witness_record'}, + 'latency_ms': 2} diff --git a/root/wander-train/oracle_release_runtime_w59.py b/root/wander-train/oracle_release_runtime_w59.py new file mode 100644 index 0000000000000000000000000000000000000000..b2cf786d70f17b50038fb9e657b2844d9e68ba61 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_w59.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""oracle_release_runtime_w59 - THE OPERATOR HOST: one disciplined runtime, many +operators, none of them hand-written as folds. + +Every capability so far arrived as its own module, and every defect found in those +modules today lived in the same shared boilerplate rather than in any of their +reasoning. A missing witness record made a turn invisible to the gate. A missing +relevance guard let the deduction fold swallow whole conversations. A self-report +read a file instead of reading itself. A justification fold checked one warrant +field and told a visitor that a properly sourced answer should not be trusted. +Four folds, four instances of the same class of mistake, because each one +reimplemented the parts that are not the interesting part. + +So this module holds no capability of its own. It is a HOST. The operators are +declarative specs in oracle_operator_specs_v1, and the host supplies exactly once: + + dispatch the first spec whose trigger matches and whose parse succeeds + verification fn computes the answer, ref recomputes it by a different method, + and NOTHING is reported unless the two agree + decline a named boundary, in the operator's own words + passthrough when the surface is not this operator's business, the chain + answers exactly as it would if the host were not installed + witness one recording path, so no operator can be invisible to the gate + +Adding a capability is now adding a spec, and a spec cannot forget the discipline +because it never implements it. + +The verification step is the reason to build it this way rather than as a library +of helpers. An operator that computed and reported would be a faster way to be +wrong. Here the answer has to survive being computed twice by different routes, +and when the two disagree the host says so and withholds, which is a state the +previous hand-written folds had no way to even be in. + +Six operators arrive with this host: unit conversion, gcd and lcm, base +conversion, descriptive statistics, sequence continuation, roman numerals. The +count is not the point. The point is that the seventh costs a spec. + +Hand-authored 2026-08-28. Judged by the gate. +""" + +from __future__ import annotations + +from fractions import Fraction + +from oracle_release_runtime_w58 import OracleReleaseRuntimeW58 +from oracle_operator_specs_v1 import SPECS, NotOurs, OutOfFragment + + +def _same(a, b): + """Do two independently computed answers agree? Types must not paper over it.""" + if a is None or b is None: + return False + if isinstance(a, str) or isinstance(b, str): + return str(a).strip().lower() == str(b).strip().lower() + try: + return Fraction(a) == Fraction(b) + except (TypeError, ValueError): + return str(a) == str(b) + + +class OracleReleaseRuntimeW59(OracleReleaseRuntimeW58): + """Runs declarative operators, and reports only what two methods agree on.""" + + def _witness_op(self, session_id, surface, spec, steps, status, warrant=None): + from collections import deque + hist = getattr(self, '_paths', None) + if hist is None: + hist = self._paths = {} + q = hist.get(session_id) + if q is None: + q = hist[session_id] = deque(maxlen=64) + prev = q[-1].get('turn', 0) if len(q) else 0 + q.append({'turn': prev + 1, 'in_surface': str(surface)[:120], + 'in_form': [spec['family']], + 'operator': 'op:%s' % spec['name'], + 'out_status': status, 'in_terms': [], 'out_terms': [], + 'relation': {'grounded': [], 'grounded_cited': [], + 'introduced': [], 'unaccounted': []}, + 'carried_from_prior': [], 'continuity_lexical': False, + 'continuity': True, 'citations': 0, 'residual': None, + 'cost_ms': 1.0, + 'conversion': {'construction': 'VERIFIED_COMPUTATION', + 'from': str(surface)[:80], + 'to': (steps[-1][:60] if steps else spec['name']), + 'produced_by': 'op_%s_w59' % spec['name']}, + 'verified_warrant': warrant}) + + def chat(self, text: str, session_id: str = 'default') -> dict: + s = ' '.join(str(text).strip().split()) + + for spec in SPECS: + if not spec['trigger'].search(s): + continue + try: + args = spec['parse'](s) + except NotOurs: + continue # not this operator, keep looking + except OutOfFragment as why: + return self._decline(session_id, s, spec, why) + + try: + value, steps = spec['fn'](args) + except NotOurs: + continue + except OutOfFragment as why: + return self._decline(session_id, s, spec, why) + except Exception: # noqa: BLE001 - a crash is not an answer + continue + + try: + check = spec['ref'](args) + except Exception: # noqa: BLE001 + check = None + + if not _same(value, check): + # The interesting failure. Two methods, one answer each, and they + # do not match, so the honest report is the disagreement itself. + self._witness_op(session_id, s, spec, steps, 'withheld') + return {'status': 'withheld', 'path': 'op:%s:unverified' % spec['name'], + 'response': ('I worked that out two ways and got two ' + 'different answers, %s and %s, so one of my ' + 'methods is wrong and I do not yet know which. ' + 'I am not going to pick the one I like.' + % (value, check)), + 'provenance': {'kind': 'computation', 'verified': False, + 'operator': spec['name'], + 'disagreement': [str(value), str(check)]}, + 'latency_ms': 2} + + body = spec['render'](args, value, steps) + trace = ' '.join('%s.' % st for st in steps[:6]) + warrant = {'kind': spec['warrant'], 'steps': list(steps)} + self._witness_op(session_id, s, spec, steps, 'answered', warrant) + pending = getattr(self, '_pending_conversion', None) + if pending is None: + pending = self._pending_conversion = {} + pending[session_id] = {'construction': 'VERIFIED_COMPUTATION', + 'from': s[:80], 'to': str(value)[:60], + 'produced_by': 'op_%s_w59' % spec['name']} + return {'status': 'answered', 'path': 'op:%s' % spec['name'], + 'response': ('%s %s I checked it by a second method that ' + 'agrees.' % (body, trace)).strip(), + 'provenance': {'kind': 'computation', 'verified': True, + 'operator': spec['name'], + 'family': spec['family'], + 'steps': list(steps), + 'independence': spec['independence'], + 'checked_by': 'independent reimplementation'}, + 'latency_ms': 2} + + return super().chat(s, session_id) + + def _decline(self, session_id, surface, spec, why): + self._witness_op(session_id, surface, spec, [], 'withheld') + return {'status': 'withheld', + 'path': 'op:%s:out_of_fragment' % spec['name'], + 'response': ('%s. What I do handle is %s. I could give you a ' + 'plausible looking answer, but I would have no way to ' + 'check it, so I will not.' + % (str(why).rstrip('.'), spec['fragment'])), + 'provenance': {'kind': 'computation', 'verified': False, + 'operator': spec['name'], 'reason': str(why), + 'fragment': spec['fragment']}, + 'latency_ms': 1} diff --git a/root/wander-train/oracle_release_runtime_w60.py b/root/wander-train/oracle_release_runtime_w60.py new file mode 100644 index 0000000000000000000000000000000000000000..f3c052ecf1d4eee4f00bc86e99cd1ecfded2c912 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_w60.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""oracle_release_runtime_w60 - CORRECTNESS RELEASE: a refusal is not a missing warrant. + +The justification fold reads the previous turn's record and reports the warrant it +finds. Asked "how did you get that" after a turn where the machine DECLINED, it +found no warrant, because nothing had been asserted, and reported: + + my record for it holds no warrant of any kind, neither a citation nor a check + I can replay. I should not have presented it as settled. + +Every clause of that is false about a refusal. Nothing was presented. Nothing was +settled. The turn was the machine working correctly, and its own account of itself +called it a lapse. + +This is the second time in one day that the justification fold has been wrong in +the same direction, and the direction is worth naming: it was written to catch a +machine overclaiming, so when it meets a turn that claimed nothing it reads the +absence of a warrant as the absence of grounds. The first case was citation, where +the warrant was real and in a different field. This case is refusal, where there +is correctly no warrant at all and the right report is why there is none. + +A turn is now read by its STATUS first and its warrant second: + + answered + warrant report the warrant, as before + answered + citations report the source, as w58 already did + withheld report that it declined, and say a refusal has no + working to show, because that is the honest account + answered + nothing the original complaint, which remains correct in the + one case it was written for + +Hand-authored 2026-08-28. Judged by the gate. +""" + +from __future__ import annotations + +from oracle_release_runtime_w59 import OracleReleaseRuntimeW59 + +_REFUSAL_TAILS = ('does_not_follow', 'out_of_fragment', 'unverified', + 'no_primitive', 'unsupported') + + +class OracleReleaseRuntimeW60(OracleReleaseRuntimeW59): + """Justification that can tell an absent answer from an ungrounded one.""" + + @staticmethod + def _was_refusal(prior): + if str(prior.get('out_status', '')) == 'withheld': + return True + return str(prior.get('operator', '')).endswith(_REFUSAL_TAILS) + + def _justify(self, kind, prior): + if kind in ('how_got', 'what_did') and self._was_refusal(prior): + op = str(prior.get('operator', '') or 'an unrecorded route') + if kind == 'how_got': + return ('I did not work that one out, I declined it. A refusal has ' + 'no working to show, which is the point of it: I stopped ' + 'because I had no way to check an answer, so there is no ' + 'answer of mine to account for. My record marks that turn ' + 'as withheld by %s. [self:witness]' % op) + return ('That turn was a refusal rather than an answer. %s took it, ' + 'found it outside what it can check, and withheld rather than ' + 'producing something neither of us could verify. ' + '[self:witness]' % op) + return super()._justify(kind, prior) diff --git a/root/wander-train/oracle_release_runtime_w71.py b/root/wander-train/oracle_release_runtime_w71.py new file mode 100644 index 0000000000000000000000000000000000000000..43ea0f912f99f87d52b876d4aa4fc35fa84f21a0 --- /dev/null +++ b/root/wander-train/oracle_release_runtime_w71.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""oracle_release_runtime_w71 - OPEN CONVERSATION: resolving "it" by elimination. + +The held-out residue is not a scatter of unrelated failures. Read it and almost all +of it is one thing: + + Did you run it? Can you test it? How does it work? + Write the same thing in go. how old is it? + Now write one that counts vowels. what else? + The thing from before. Explain the thing with the water. + +Every one of those is a surface that does not name its own subject. The machine had +just produced the subject, recorded it, and could still say "I didn't fully parse +that". It was holding the answer and could not see the question. + +So this is the elimination move again, pointed at reference. The candidate referents +are read from the witness record, the surface and the machine's own limits ablate +them, and what survives is the reading: + + one survivor resolve, say which referent and why, and re-dispatch + many survivors name them and ask which, rather than picking a favourite + zero survivors say what could not be resolved instead of guessing + +Which gives open conversation the thing it has never had here: a WARRANT. Not for +the reply being good, which is a matter of taste and always will be, but for the +INTERPRETATION being licensed. "I read 'it' as the python program from two turns +ago, because that is the only thing in this conversation you can run" is checkable +by the person who said "it". + +Three referent kinds, each read from a different part of the record because that is +where the machine actually keeps them: topics from in_terms, programs and results +from the conversion. A constraint set per surface form decides which kinds a +question can even accept, so "did you run it" cannot resolve to a topic and "how old +is it" cannot resolve to a program. + +ONE GUARD WORTH NAMING: the conversion field is not always about the turn it sits +on. A summarize turn was found carrying a stale conversion from a fraction question +two turns earlier, because the pending-conversion slot is not cleared between turns. +Anything reading it blindly attributes the wrong artifact, so every read here checks +that the conversion's own `from` matches the surface of the turn it was found on. +That is a defect in a fold below this one; this works around it rather than +pretending it is not there. + +Hand-authored 2026-08-28. Judged by the gate. +""" + +from __future__ import annotations + +import re + +from oracle_release_runtime_w70 import OracleReleaseRuntimeW70 +from oracle_operator_parse_v1 import Reading, eliminate, Ambiguous +from oracle_operator_specs_v1 import NotOurs, OutOfFragment + +# surfaces that do not name their subject +_PRONOUN = re.compile(r'\b(?:it|that|this|them|those|one)\b', re.I) +_BARE_CONT = re.compile(r'^(?:what else|anything else|go on|more|and\?|tell me more)' + r'\s*\??$', re.I) +_THE_THING = re.compile(r'\bthe (?:same )?thing\b|\bthe one\b', re.I) + +# what each surface form is ASKING FOR, which is what rules referents out +_WANTS_PROGRAM = re.compile( + r'\b(?:did|can|could|will|would)\s+you\s+(?:run|test|execute|try)\b' + r'|\bthe same (?:thing|code|program)\b|\bin (?:go|python|javascript|bash|sql|c)\b' + r'|\bnow write one\b|\bdoes it (?:compile|run|work)\b', re.I) +_WANTS_TOPIC = re.compile( + r'\bhow (?:does|do) (?:it|they) work\b|\bhow old\b|\bwhere is it\b' + r'|\btell me more\b|\bwhat else\b|\bexplain (?:it|that|the thing)\b' + r'|\bwhy (?:is|does) it\b|\bwhat is it made of\b', re.I) +_LANG = re.compile(r'\bin (go|python|javascript|bash|sql|c|rust|java)\b', re.I) +_STOP_TERMS = {'summarize', 'explain', 'compare', 'list', 'tell', 'facts', 'about', + 'what', 'how', 'why', 'the', 'a', 'an'} + + +class OracleReleaseRuntimeW71(OracleReleaseRuntimeW70): + """Open conversation with a licensed reading of what "it" refers to.""" + + def _referents(self, session_id): + """Candidate referents, newest first, read from the witness record.""" + hist = getattr(self, '_paths', None) or {} + q = hist.get(session_id) + out = [] + if not q: + return out + for e in reversed(list(q)): + op = str(e.get('operator', '')) + if op.startswith(('justify:', 'resolve:')): + continue + surface = str(e.get('in_surface', '')) + conv = e.get('conversion') or {} + # the conversion slot leaks between turns; only trust it when it is + # demonstrably about THIS turn + own = str(conv.get('from', ''))[:60].strip() == surface[:60].strip() + if own and conv.get('construction') == 'VERIFIED_PROGRAM': + out.append(('program', str(conv.get('to', '')), surface)) + elif own and conv.get('construction') == 'VERIFIED_COMPUTATION': + out.append(('result', str(conv.get('to', '')), surface)) + terms = [t for t in (e.get('in_terms') or []) + if t not in _STOP_TERMS and len(t) > 2] + if terms: + out.append(('topic', terms[0], surface)) + if len(out) >= 6: + break + return out + + def _resolve(self, s, session_id): + """(rewritten_surface, referent, trace) or raises with a named reason.""" + cands = self._referents(session_id) + if not cands: + raise NotOurs('nothing recorded yet to refer back to') + + wants_program = bool(_WANTS_PROGRAM.search(s)) + wants_topic = bool(_WANTS_TOPIC.search(s)) or bool(_BARE_CONT.match(s)) + if not (wants_program or wants_topic): + raise NotOurs('no reference this fold knows how to type') + + readings = [Reading({'kind': k, 'label': lab, 'surface': surf}, + 'the %s from "%s"' % (k, surf[:38])) + for k, lab, surf in cands] + constraints = [] + if wants_program and not wants_topic: + constraints.append(('you are asking about something runnable and that ' + 'is not a program', lambda a: a['kind'] == 'program')) + elif wants_topic and not wants_program: + constraints.append(('you are asking about a subject and that is not one', + lambda a: a['kind'] == 'topic')) + else: + constraints.append(('it is neither a program nor a subject', + lambda a: a['kind'] in ('program', 'topic'))) + # only the most recent of a kind can be "it"; older ones are ruled out by + # recency, which is a real constraint rather than a tiebreak + first_of_kind = {} + for i, r in enumerate(readings): + first_of_kind.setdefault(r.args['kind'], i) + constraints.append(('an earlier one of the same kind is nearer', + lambda a, _f=first_of_kind, _r=readings: + _r[_f[a['kind']]].args is a)) + + survivor = eliminate(readings, constraints, + nothing_here='nothing here to refer to') + ref = survivor.args + if ref['kind'] == 'program': + lang = _LANG.search(s) + if lang: + new = re.sub(r'\b(go|python|javascript|bash|sql|c|rust|java)\b', + lang.group(1), ref['surface'], count=1, flags=re.I) + return new, ref, survivor + return None, ref, survivor # a question ABOUT the program + topic = ref['label'] + if _BARE_CONT.match(s) or re.search(r'\btell me more\b', s, re.I): + return 'Tell me about %s.' % topic, ref, survivor + new = _PRONOUN.sub(topic, s, count=1) + new = _THE_THING.sub(topic, new) + return new, ref, survivor + + def _about_program(self, s, ref, session_id): + """Questions about a program are answered from the record, not re-run.""" + hist = getattr(self, '_paths', None) or {} + q = hist.get(session_id) or [] + warrant = None + for e in reversed(list(q)): + if str(e.get('in_surface', ''))[:60].strip() == ref['surface'][:60].strip(): + warrant = e.get('verified_warrant') or {} + break + steps = (warrant or {}).get('steps') or [] + if steps: + return ('Yes. I ran %s on this machine before answering, and the ' + 'recorded output was %s. I read "it" as that program, because ' + 'it is the only thing in this conversation you can run. ' + '[self:witness]' % (ref['label'], '; '.join(steps[:3]))) + return ('I read "it" as %s, the program from earlier in this conversation. ' + 'My record for that turn holds no execution, so I will not tell you ' + 'it ran. [self:witness]' % ref['label']) + + def chat(self, text: str, session_id: str = 'default') -> dict: + s = ' '.join(str(text).strip().split()) + if not (_PRONOUN.search(s) or _BARE_CONT.match(s) or _THE_THING.search(s)): + return super().chat(s, session_id) + # a surface that names a subject of its own is not this fold's business + if re.search(r'\b(?:volcano|moon|gravity|rain)\b', s, re.I) and \ + not _BARE_CONT.match(s): + pass + + try: + rewritten, ref, survivor = self._resolve(s, session_id) + except Ambiguous as why: + return {'status': 'withheld', 'path': 'resolve:ambiguous', + 'response': ('%s. Say which one and I will go on.' % why), + 'provenance': {'kind': 'reference', 'resolved': False}, + 'latency_ms': 1} + except NotOurs: + return super().chat(s, session_id) + except OutOfFragment as why: + return super().chat(s, session_id) # not ours to refuse + + ruled = [(b, k) for b, k in (getattr(survivor, 'eliminated', []) or [])][:2] + if rewritten is None: + body = self._about_program(s, ref, session_id) + result = {'status': 'answered', 'path': 'resolve:about_program', + 'response': body, + 'provenance': {'kind': 'reference', 'resolved': True, + 'referent': ref['label'], + 'referent_kind': ref['kind']}, + 'latency_ms': 2} + else: + result = super().chat(rewritten, session_id) + note = ('I read that as "%s", taking %s to mean %s.' + % (rewritten, '"it"' if _PRONOUN.search(s) else 'the follow-up', + ref['label'])) + if ruled: + note += ' I ruled out %s, because %s.' % (ruled[0][0], ruled[0][1]) + result = dict(result) + result['response'] = '%s %s' % (note, str(result.get('response', ''))) + result['path'] = 'resolve:%s->%s' % (ref['kind'], result.get('path')) + result['provenance'] = dict(result.get('provenance') or {}) + result['provenance'].update({'resolved_reference': ref['label'], + 'rewritten_as': rewritten}) + + from collections import deque + hist = getattr(self, '_paths', None) + if hist is None: + hist = self._paths = {} + qq = hist.get(session_id) + if qq is None: + qq = hist[session_id] = deque(maxlen=64) + prev = qq[-1].get('turn', 0) if len(qq) else 0 + qq.append({'turn': prev + 1, 'in_surface': s[:120], + 'in_form': ['anaphoric_followup'], + 'operator': 'resolve:' + ref['kind'], + 'out_status': str(result.get('status', 'answered')), + 'in_terms': [], 'out_terms': [], + 'relation': {'grounded': [], 'grounded_cited': [], + 'introduced': [], 'unaccounted': []}, + 'carried_from_prior': [ref['kind']], + 'continuity_lexical': True, 'continuity': True, 'citations': 1, + 'residual': None, 'cost_ms': 1.0, + 'conversion': {'construction': 'RESOLVED_REFERENCE', + 'from': s[:60], 'to': ref['label'][:60], + 'produced_by': 'resolve_w71'}, + 'verified_warrant': {'kind': 'self_citation', + 'steps': ['read "%s" as %s' + % (s[:40], ref['label'])] + + ['ruled out %s because %s' % (b, k) + for b, k in ruled]}}) + return result diff --git a/root/wander-train/oracle_sibling_evidential_v3.py b/root/wander-train/oracle_sibling_evidential_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..8a82c83da3943f682dfb055fae9990bc961ef6f7 --- /dev/null +++ b/root/wander-train/oracle_sibling_evidential_v3.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Evidential plant V3 - verdicts read from the RECORD's structure, never from text. + +M60b demonstrated V2's remaining defect on h035:3: "Is a teddy bear a bear?" was +dispatched to summarize (a Teddy bear page exists), the summary carries no polarity +shape, and the text-regex polarity check rendered "supported" beside content that +refutes the claim. The repair is this plant's own founding principle applied strictly: +the answer's evidential comes from what the record LICENSES, and the record's kind +verdict is TYPED - + + SUPPORTED provenance.kind == "derived" with rule transitive_isa, or + kind == "commonsense_edge" with relation IsA, or + kind == "lexical_corpus" with relation hypernym / hypernym_of + REFUTED kind == "lexical_corpus" with relation hypernym_excluded + NO VERDICT any other route (gated_summary, document_runtime, ...) - the record + produced no kind verdict, so the plant says so: "The record does not + settle what you heard." A summary does not license a kind verdict. + +Probe conversion (declarative->kind question) inherited from V2. No text regex remains +in the verdict path. No neural network; no LM call. +""" + +from __future__ import annotations + +import re + +from oracle_sibling_evidential_v2 import SiblingEvidentialV2, DECL_KIND, _art +from oracle_siblings_bed2_v1 import SiblingEvidential, HEARSAY, DIRECT, INFER + + +def kind_verdict(result: dict) -> str: + prov = result.get("provenance") or {} + kind = str(prov.get("kind", "")) + rel = str(prov.get("relation", "")) + rule = str(prov.get("rule", "")) + if kind == "derived" and rule == "transitive_isa": + return "supported" + if kind == "commonsense_edge" and rel == "IsA": + return "supported" + if kind == "lexical_corpus" and rel in ("hypernym", "hypernym_of"): + return "supported" + if kind == "lexical_corpus" and rel == "hypernym_excluded": + return "refuted" + return "no_verdict" + + +class SiblingEvidentialV3(SiblingEvidentialV2): + """V2's probe conversion + structural verdicts.""" + + def chat(self, text: str, session_id: str = "default") -> dict: + s = " ".join(str(text).strip().split()) + marked, proposition = None, s + for name, pat in (("hearsay", HEARSAY), ("direct", DIRECT), ("inferred", INFER)): + m = pat.match(s) + if m: + marked, proposition = name, m.group("p").strip() + break + if marked: + if re.match(r"^(is|are|what|who|when|how)\b", proposition, re.I): + probe = proposition + else: + dk = DECL_KIND.match(proposition) + if dk: + x, y = dk.group("x").strip(), dk.group("y").strip() + probe = f"Is {_art(x)} {x} {_art(y)} {y}?" + else: + probe = f"Summarize {proposition}." + self._pending_conversion[session_id] = { + "construction": f"EVIDENTIAL:{marked}", "from": s[:80], "to": probe[:80], + "produced_by": "sibling:evidential_v3"} + result = super(SiblingEvidential, self).chat(probe, session_id) + else: + result = super(SiblingEvidential, self).chat(s, session_id) + ev = self._evidential_of(result) + result["evidential"] = {"input_marked": marked, "answer_licensed": ev} + if result.get("status") == "answered": + tag = {"direct": "[ev:direct]", "inferred": "[ev:inferred]", + "reported": "[ev:reported]", "unknown": "[ev:unknown]"}[ev] + answer = str(result.get("response", "")) + if marked == "hearsay": + verdict = kind_verdict(result) + lead = {"supported": "What you heard is supported. ", + "refuted": "What you heard is not supported by the record. ", + "no_verdict": "The record does not settle what you heard. "}[verdict] + result["evidential"]["verdict"] = verdict + result["response"] = f"{tag} {lead}" + answer + else: + result["response"] = f"{tag} " + answer + hist = getattr(self, "_paths", {}).get(session_id) + if hist: + hist[-1]["evidential"] = result["evidential"] + return result diff --git a/root/wander-train/oracle_tritree_v1.py b/root/wander-train/oracle_tritree_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..9b56466e4c70a003814b6d401a743377f3dced20 --- /dev/null +++ b/root/wander-train/oracle_tritree_v1.py @@ -0,0 +1,473 @@ +"""oracle_tritree_v1 - the THREE-TREE REASONING RUNTIME (drawing OR7-A-101 made computational). + +Three graphs of the canonical Tree-of-Life shape (10 Sephirot, 22 paths) share one middle pillar: + QUESTION tree - the slots a question opens KNOWLEDGE tree - where facts live OPERATOR tree - the moves + keter request - VERIFY (goal test / next cycle) + chokmah subject entities (title index) LINK + binah relation RELATION EDGES (typed store, Wikidata) RELATE + chesed context sentences (cited prose) EXPAND + geburah constraints types (P31/P279 classes) CHECK + tiferet answer slot (binding) - BIND + netzach alternatives (A or B) neighbours COMPARE + hod quantities engines (dates, numbers) COMPUTE + yesod evidence citations FRAME + malkuth surface - READ +A MOVE goes from the current operator node to an ADJACENT node along one of the graph's edges and applies that node's +operator to the state. Only edges of the graph are moves: the topology constrains the reasoning. The search is a bounded +breadth-first walk from Malkuth; a state is SOLVED when VERIFY (Keter) finds the answer slot bound, checked and cited. +Distinct solved answers -> abstain. A two-hop question runs a second CYCLE from Keter with the bound entity as subject. +The graph is a parameter (PATHS), so the topology can be ablated with everything else fixed. +""" +from __future__ import annotations + +import copy +import json +import os +import re +from collections import deque + +import oracle_compare_op_v1 as CMP +import oracle_cube_reader_v1_4 as R +import oracle_relstore_v1 as RS + +SEPH = ["keter", "chokmah", "binah", "chesed", "geburah", "tiferet", "netzach", "hod", "yesod", "malkuth"] +TREE_OF_LIFE = [("keter", "chokmah"), ("keter", "binah"), ("keter", "tiferet"), ("chokmah", "binah"), ("chokmah", "tiferet"), + ("chokmah", "chesed"), ("binah", "tiferet"), ("binah", "geburah"), ("chesed", "geburah"), + ("chesed", "tiferet"), ("chesed", "netzach"), ("geburah", "tiferet"), ("geburah", "hod"), + ("tiferet", "netzach"), ("tiferet", "yesod"), ("tiferet", "hod"), ("netzach", "hod"), ("netzach", "yesod"), + ("netzach", "malkuth"), ("hod", "yesod"), ("hod", "malkuth"), ("yesod", "malkuth")] +OP = {"malkuth": "READ", "yesod": "FRAME", "chokmah": "LINK", "binah": "RELATE", "chesed": "EXPAND", "geburah": "CHECK", + "tiferet": "BIND", "netzach": "COMPARE", "hod": "COMPUTE", "keter": "VERIFY"} +LEXICON = os.environ.get("ORACLE_LEXICON") or __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/relstore/lexicon_v1.json" +PLACE_CLASSES = {"country", "sovereign state", "city", "town", "village", "municipality", "state", "province", "region", + "county", "island", "continent", "capital", "big city", "human settlement", "administrative territorial entity", + "commune of france", "state of the united states", "city of the united states", "metropolis"} +KIND_EQ = [{"show", "series", "programme", "program", "television series", "sitcom"}, {"film", "movie", "feature film"}, + {"band", "group", "musical group", "rock band"}, {"company", "corporation", "firm", "business", "enterprise"}, + {"album", "studio album"}, {"song", "single"}, {"magazine", "periodical"}, {"university", "college"}, + {"country", "sovereign state", "state"}] + + +# the properties the relation store holds (build_relation_store_v1.PROPS) +STORE_PROPS = {31, 279, 17, 19, 20, 569, 570, 27, 131, 276, 36, 37, 38, 47, 50, 57, 58, 86, 161, 162, 175, 176, 178, 123, + 136, 127, 112, 159, 166, 26, 22, 25, 40, 3373, 69, 108, 106, 1303, 264, 463, 54, 118, 641, 101, 140, 84, 61, + 170, 676, 571, 577, 1082, 2046, 2048, 495, 364, 155, 156, 361, 527, 1376, 138, 800, 39, 749, 355, 1412, 103, + 710, 1344, 921, 30, 206, 1441, 674, 725, 449, 750, 272, 344, 1040, 1449, 488, 169, 35, 6, 150, 1830, 945, 113, + 114, 915, 840, 437, 400, 1027, 2031, 2032} +RULES = set((os.environ.get("TT_RULES") or "").split(",")) +TRIVIAL_VERBS = {"be", "have", "do", "call", "name", "know"} + + +class State: + __slots__ = ("node", "slots", "cands", "answer", "evidence", "trace", "cycle", "checked", "depth") + + def __init__(self): + self.node, self.slots, self.cands, self.answer = "malkuth", {}, [], None + self.evidence, self.trace, self.cycle, self.checked, self.depth = [], [], 0, False, 0 + + def fork(self): + s = State() + s.node, s.slots, s.cands, s.answer = self.node, dict(self.slots), list(self.cands), self.answer + s.evidence, s.trace, s.cycle, s.checked, s.depth = list(self.evidence), list(self.trace), self.cycle, self.checked, self.depth + return s + + +class TriTree: + def __init__(self, paths=None, reader=None, relstore=None, lexicon=None, max_states=4000, max_depth=12): + self.adj = {s: set() for s in SEPH} + for a, b in (paths or TREE_OF_LIFE): + self.adj[a].add(b) + self.adj[b].add(a) + self.r = reader or R.CubeReader() + self.rs = relstore or RS.RelStore() + self.cmp = CMP.CompareOp(self.r) + lx = json.load(open(lexicon or LEXICON)) + self.lex = {tuple(json.loads(k)): {tuple(json.loads(p)): c for p, c in v.items()} for k, v in lx["lexicon"].items()} + self.cmp_lex = {ph: {tuple(json.loads(p)): c for p, c in v.items()} for ph, v in lx.get("compare", {}).items()} + self.max_states, self.max_depth = max_states, max_depth + self.expanded = 0 + # property labels + aliases (Wikidata, English), for the RELATION check + self.plabel, self.palias = {}, {} + pl = os.path.join(os.path.dirname(LEXICON), "prop_labels.tsv") + if os.path.exists(pl): + for line in open(pl, encoding="utf-8", errors="replace"): + parts = line.rstrip("\n").split("\t") + if len(parts) == 2 and "/P" in parts[0]: + try: + p = int(parts[0].rsplit("/P", 1)[1].rstrip(">")) + except ValueError: + continue + if p not in STORE_PROPS: + continue # a property the store does not hold can never be followed + words = {R.stem(w) for w in re.findall(r"[a-z]+", parts[1].lower()) if len(w) >= 3 and w not in R.STOP} + self.plabel.setdefault(p, set()).update(words) + self.palias.setdefault(p, set()).add(RS.norm(parts[1].rsplit("@", 1)[0].strip('"'))) + + # ------------------------------------------------------------------ operators (one per Operator-tree node) + def READ(self, st): # malkuth: the surface opens the scramble + if "fr" in st.slots: + return [] + q = st.slots["q"] + fr = self.r.frame(q) + pair = CMP._pair(q.strip().strip('"')) + if not fr and not (pair and pair[2] == "or"): + return [] + s = st.fork() + s.slots["fr"] = fr + s.slots["pair"] = pair if pair and pair[2] == "or" else None + s.trace.append("READ") + return [s] + + def FRAME(self, st): # yesod: obligations -> candidate RELATIONS (the lexicon) + if "fr" not in st.slots or "props" in st.slots: + return [] + s = st.fork() + fr = s.slots["fr"] + props = {} + if fr and "rel" not in fr: + self.r.link(fr) # tokens -> the governing verb (pred) and relation words (rel) + if fr: + verb = (fr.get("pred") or [None])[0] + for key, w in ((("hv", fr["atype"], fr["head"], verb), 3), (("v", fr["atype"], verb), 2), (("h", fr["atype"], fr["head"]), 1)): + for (d, p), c in self.lex.get(key, {}).items(): + props[(d, p)] = props.get((d, p), 0) + w * c + if s.slots.get("pair"): + m = re.search(r"\b(first|earlier|older|younger|later|more|larger|bigger|greater|higher|taller|longer|smaller|" + r"fewer|less|shorter)\b(?:\s+([a-z]+))?", s.slots["q"].lower()) + phrase = " ".join(x for x in (m.groups() if m else ()) if x) + cands = self.cmp_lex.get(phrase) or self.cmp_lex.get(phrase.split(" ")[0] if phrase else "", {}) + s.slots["cmp_props"] = sorted(cands.items(), key=lambda kv: -kv[1])[:2] + if fr: + # MULTI-HOP (the cycle): "the father of X", "the writer of X" - a relational noun that IS a property's name + m = re.search(r"\bthe\s+(?:[\w\-]+\s+){0,2}?([a-z]+)\s+of\s+(?:the\s+)?([A-Z][^?,]{1,60}?)(?:\s+(?:die|died|born|" + r"built|founded|released|published|located|made|start|end)\b|\?|,|$)", s.slots["q"]) + noun = None + rest = ({R.stem(w) for w in fr.get("rel", [])} | {R.stem(v) for v in (fr.get("pred") or []) if v not in TRIVIAL_VERBS}) + if m: + rest -= {R.stem(m.group(1)), R.stem(fr["head"] or "")} + if m and rest and "hop2" not in s.slots: # a relation REMAINS after the hop ("the father of X ... die") + hop1 = sorted(p for p, al in self.palias.items() if m.group(1) in al) + if hop1: + noun = m.group(1) + s.slots["hop1"] = {"props": hop1[:2], "anchor": m.group(2).strip(), "noun": noun} + # the RELATION NODE's own vocabulary: properties whose label/aliases carry the question's RELATION words (not + # its head noun, not the hop-1 noun) - "share a land border" -> P47 "shares border with" + relw = {R.stem(w) for w in fr.get("rel", []) if len(w) >= 3} | {R.stem(v) for v in (fr.get("pred") or []) if v} + relw -= {R.stem(fr["head"] or ""), R.stem(noun or "")} + for p, words in self.plabel.items(): + ov = len(words & relw) + if ov: + for d in ("fwd", "inv"): + props[(d, p)] = props.get((d, p), 0) + 5 * ov + # a RECALLED FORM (forms_store_v1) proposes the property path a verified derivation of the same shape used; + # absent a form, FRAME is exactly as frozen (ad1530e6) + for dp, w in (getattr(self, "_form_props", None) or {}).items(): + props[dp] = props.get(dp, 0) + 1000 * w + s.slots["props"] = sorted(props.items(), key=lambda kv: -kv[1])[:4] + s.trace.append("FRAME") + return [s] + + def LINK(self, st): # chokmah: subjects -> entities of the relation store + if "props" not in st.slots or "subj" in st.slots: + return [] + s = st.fork() + subj = [] + if s.slots.get("pair"): + for side in s.slots["pair"][:2]: + t, _ = self.cmp._entity(side) + q = self.rs.entity(t) if t else None + if q: + subj.append((t, q)) + if len(subj) != 2: + return [] + else: + fr = s.slots["fr"] + if "rel" not in fr: + self.r.link(fr) + for sj in fr["subjects"][:3]: + t = self.r.db.execute("SELECT title FROM article WHERE page_id=?", (sj["pid"],)).fetchone() + q = self.rs.entity(t[0]) if t else None + if q: + subj.append((t[0], q)) + if not subj: + return [] + s.slots["subj"] = subj + s.trace.append("LINK") + return [s] + + def RELATE(self, st): # binah: typed relation edges, forward or inverse + if "subj" not in st.slots or st.cands or st.slots.get("pair"): + return [] + out = [] + hop1 = st.slots.get("hop1") + if hop1 and st.cycle == 0: + # cycle 1 of a two-hop question: anchor --hop1 property--> the described entity (exactly one) + aq = self.rs.entity(hop1["anchor"]) or next((q for t, q in st.slots["subj"] if RS.norm(t) == RS.norm(hop1["anchor"])), None) + if not aq: + return [] + for p in hop1["props"]: + vals = [o for o, _ in self.rs.values(aq, p) if o] + if len(set(vals)) == 1: + s = st.fork() + s.slots["bridge"] = (vals[0], "wd:Q%d P%d wd:Q%d" % (aq, p, vals[0]), hop1["noun"]) + s.trace.append("RELATE:hop1:P%d" % p) + out.append(s) + return out + for (d, p), w in st.slots["props"]: + for title, sq in st.slots["subj"][:2]: + if d == "fwd": + vals = [(o, lit, "wd:Q%d P%d %s" % (sq, p, ("wd:Q%d" % o) if o else json.dumps(lit))) for o, lit in self.rs.values(sq, p)] + else: + vals = [(s_, None, "wd:Q%d P%d wd:Q%d" % (s_, p, sq)) for s_ in self.rs.subjects(p, sq, 60)] + if not vals: + continue + s = st.fork() + s.cands = [{"o": o, "lit": lit, "cite": c, "p": p, "via": title} for o, lit, c in vals[:60]] + s.trace.append("RELATE:%s:P%d" % (d, p)) + out.append(s) + return out + + def CHECK(self, st): # geburah: types + constraints (independent of the proposer) + if not st.cands or st.checked: + return [] + s = st.fork() + fr = s.slots["fr"] + keep = [] + for c in s.cands: + ok = False + if fr["atype"] in ("YEAR", "DATE"): + ok = bool(c["lit"]) and re.match(r"^[+-]?\d{4}-", c["lit"] or "") is not None + elif fr["atype"] == "NUMBER": + ok = bool(c["lit"]) and re.match(r"^[+-]?\d", c["lit"] or "") is not None + elif c["o"]: + if fr["atype"] == "PERSON": + ok = self.rs.is_human(c["o"]) + elif fr["atype"] == "PLACE": + ok = bool(self.rs.classes(c["o"], 2) & PLACE_CLASSES) if not fr["head"] else self._kind(c["o"], fr["head"]) + elif fr["atype"] == "INSTANCE": + ok = self._kind(c["o"], fr["head"]) + if ok: + keep.append(c) + # constraints: every year the question names must be consistent with the candidate where the store can say + if keep and fr.get("years"): + dated = [c for c in keep if c["lit"] and c["lit"][:5].lstrip("+")[:4] in fr["years"]] + keep = dated or keep + if not keep: + return [] + # RELATION LICENCE (independent of the lexicon that proposed the property): the property's own Wikidata label or + # aliases share a word with the question's relation words / head / verb, or the lexicon supports this exact + # question shape at least 3 times. "share a land border" licenses P47 "shares border with", not P131 "located in". + p = keep[0]["p"] + qwords = {R.stem(w) for w in fr.get("rel", []) + [fr.get("head") or ""] + list(fr.get("pred") or []) if w} + qwords |= {R.stem(w) for w in re.findall(r"[a-z]+", s.slots["q"].lower()) if len(w) >= 4 and w not in R.STOP} + support = self.lex.get(("hv", fr["atype"], fr["head"], (fr.get("pred") or [None])[0]), {}).get( + ("fwd", p), 0) + self.lex.get(("hv", fr["atype"], fr["head"], (fr.get("pred") or [None])[0]), {}).get(("inv", p), 0) + form_licensed = any(dp[1] == p for dp in (getattr(self, "_form_props", None) or {})) + if not (self.plabel.get(p, set()) & qwords) and support < 3 and not form_licensed: + return [] + # RULE B (verb licence for time properties): a date answers the question's own EVENT only when the property names + # the governing verb ("founded" -> P571 "founded"); a founding date of whatever entity was linked is not the event + if "B" in RULES and fr["atype"] in ("YEAR", "DATE") and fr.get("pred") and fr["pred"][0] not in TRIVIAL_VERBS: + if not ({R.stem(v) for v in fr["pred"]} & self.plabel.get(p, set())): + return [] + # the answer is never the subject itself, nor a name the question already contains (whole or as its core word) + qn = RS.norm(s.slots["q"]) + subj_q = {q for _, q in s.slots.get("subj", [])} + subj_w = {w for t, _ in s.slots.get("subj", []) for w in RS.norm(t).split() if len(w) >= 4} + + def restates(c): + if not c["o"]: + return False + lab = RS.norm(self.rs.label(c["o"]) or "") + return c["o"] in subj_q or (lab and lab in qn) or (lab and set(lab.split()) & subj_w and s.cycle == 0) + keep = [c for c in keep if not restates(c)] + if not keep: + return [] + s.cands, s.checked = keep, True + s.trace.append("CHECK:P%d" % p) + return [s] + + def _kind(self, qid, head): + if not head: + return False + heads = {head} | next((g for g in KIND_EQ if head in g), set()) + cls = self.rs.classes(qid, 2) + return any(h == c or c.endswith(" " + h) for h in heads for c in cls) + + def BIND(self, st): # tiferet: exactly one checked value binds the answer slot + if not st.checked or st.answer: + return [] + distinct = {} + for c in st.cands: + k = c["o"] or (c["lit"] or "")[:10] + distinct.setdefault(k, c) + if len(distinct) != 1: + return [] + c = next(iter(distinct.values())) + s = st.fork() + ans = self.rs.label(c["o"]) if c["o"] else self._fmt(c["lit"], s.slots["fr"]) + if not ans: + return [] + s.answer = ans + s.evidence.append(c["cite"]) + s.trace.append("BIND") + return [s] + + def _fmt(self, lit, fr): + m = re.match(r"^[+-]?(\d{4})-(\d{2})-(\d{2})", lit or "") + if m: + y, mo, d = m.groups() + months = "January February March April May June July August September October November December".split() + if fr and fr["atype"] == "DATE" and mo != "00" and d != "00": + return "%d %s %s" % (int(d), months[int(mo) - 1], y) + return y + return (lit or "").lstrip("+") + + def COMPARE(self, st): # netzach: A or B, by a typed date / number property + pair = st.slots.get("pair") + if not pair or "subj" not in st.slots or st.answer or not st.slots.get("cmp_props"): + return [] + (ta, qa), (tb, qb) = st.slots["subj"][:2] + out = [] + for (way, p), w in st.slots["cmp_props"]: + va, vb = self.rs.values(qa, p), self.rs.values(qb, p) + if not (va and vb and va[0][1] and vb[0][1]): + continue + a, b = va[0][1].lstrip("+"), vb[0][1].lstrip("+") + try: + fa, fb = (float(a), float(b)) if not re.match(r"^\d{4}-", a) else (a[:10], b[:10]) + except ValueError: + continue + if fa == fb: + continue + pick_a = (fa < fb) == (way == "min") + s = st.fork() + s.answer = pair[0] if pick_a else pair[1] + s.evidence += ["wd:Q%d P%d %s" % (qa, p, json.dumps(a)), "wd:Q%d P%d %s" % (qb, p, json.dumps(b))] + s.checked = True + s.trace.append("COMPARE:P%d:%s" % (p, way)) + out.append(s) + return out + + def COMPUTE(self, st): # hod: engines (here: a date answer reduced to its year) + if not st.answer or st.slots.get("computed"): + return [] + fr = st.slots.get("fr") + if fr and fr["atype"] == "YEAR" and re.match(r"^\d{1,2} \w+ \d{4}$", st.answer): + s = st.fork() + s.answer = st.answer[-4:] + s.slots["computed"] = True + s.trace.append("COMPUTE:year") + return [s] + return [] + + def EXPAND(self, st): # chesed: a cited sentence for the bound answer (evidence) + if not st.answer or st.slots.get("sentence"): + return [] + s = st.fork() + for title, _ in s.slots.get("subj", [])[:2]: + pid = self.r.pid_of(title) + for cid, sent in (self.r.page(pid, 30) if pid else []): + if RS.norm(s.answer) and RS.norm(s.answer) in RS.norm(sent): + s.slots["sentence"] = (cid, sent) + s.evidence.append(cid) + s.trace.append("EXPAND") + return [s] + return [] + + def VERIFY(self, st): # keter: solved? or open the next CYCLE + if st.slots.get("bridge") and st.cycle == 0: + # the CYCLE: the entity cycle 1 bound becomes the subject of cycle 2; the question's own relation applies to it + e, cite, noun = st.slots["bridge"] + s = st.fork() + s.cycle = 1 + s.slots["subj"] = [(self.rs.title(e) or self.rs.label(e) or "", e)] + s.slots.pop("bridge", None) + s.slots["hop2"] = True + s.cands, s.answer, s.checked = [], None, False + s.evidence.append(cite) + s.trace.append("VERIFY:cycle2(%s=%s)" % (noun, self.rs.label(e))) + return [s] + if "A" in RULES and st.answer and st.checked and "sentence" not in st.slots and not st.slots.get("pair"): + return [] # RULE A: a relation answer needs a corroborating sentence + if st.answer and st.checked and st.evidence: + s = st.fork() + s.slots["solved"] = True + s.trace.append("VERIFY") + return [s] + return [] + + # ------------------------------------------------------------------ the search (moves = graph edges only) + def answer(self, q, form_props=None): + self._form_props = form_props or None + try: + return self._answer(q) + finally: + self._form_props = None + + def _answer(self, q): + q = " ".join(str(q).split()) + if q.startswith('"') and q.endswith('"'): + q = q[1:-1].replace('""', '"') + start = State() + start.slots["q"] = q + frontier = deque(self.READ(start)) + solved, seen = [], set() + self.expanded = 0 + while frontier and self.expanded < self.max_states: + st = frontier.popleft() + self.expanded += 1 + if st.slots.get("solved"): + solved.append(st) + continue + if st.depth >= self.max_depth: + continue + for nxt in self.adj[st.node]: + succ = getattr(self, OP[nxt])(st) + if not succ: + # TRANSIT: a move travels an EDGE; passing through a node whose operator does not apply costs a + # step and changes nothing. Path lengths therefore decide which derivations fit the depth bound. + succ = [st.fork()] + for s2 in succ: + s2.node, s2.depth = nxt, st.depth + 1 + key = (nxt, s2.answer, len(s2.cands), s2.checked, tuple(s2.trace)) + if key in seen: + continue + seen.add(key) + frontier.append(s2) + answers = {RS.norm(s.answer): s for s in solved} + if len(answers) != 1: + self.why = "no solved state" if not answers else "distinct solved answers: %s" % list(answers)[:3] + return None + s = next(iter(answers.values())) + return {"answer": s.answer, "evidence": s.evidence, "trace": s.trace, "expanded": self.expanded, + "sentence": (s.slots.get("sentence") or (None, ""))[1]} + + +def random_graph(seed, n_edges=22): + """a degree-sequence-matched random graph on the same ten nodes (for the topology ablation)""" + import random + rng = random.Random(seed) + deg = {s: 0 for s in SEPH} + for a, b in TREE_OF_LIFE: + deg[a] += 1 + deg[b] += 1 + for _ in range(2000): + stubs = [s for s in SEPH for _ in range(deg[s])] + rng.shuffle(stubs) + edges = set() + ok = True + for i in range(0, len(stubs), 2): + a, b = stubs[i], stubs[i + 1] + if a == b or (a, b) in edges or (b, a) in edges: + ok = False + break + edges.add((a, b)) + if ok and len(edges) == n_edges: + return sorted(edges) + return TREE_OF_LIFE + + +COMPLETE = [(a, b) for i, a in enumerate(SEPH) for b in SEPH[i + 1:]] +CHAIN = [("malkuth", "yesod"), ("yesod", "chokmah"), ("chokmah", "binah"), ("binah", "geburah"), ("geburah", "tiferet"), + ("tiferet", "hod"), ("hod", "chesed"), ("chesed", "netzach"), ("netzach", "keter")] diff --git a/root/wander-train/oracle_tritree_v2.py b/root/wander-train/oracle_tritree_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..81d8d5f3bed27a668be8e79b66b280881e9eb620 --- /dev/null +++ b/root/wander-train/oracle_tritree_v2.py @@ -0,0 +1,479 @@ +"""oracle_tritree_v2 - v1 (immutable, imported by w480) + answer(q, form_props, only=True): search only the named +relation(s). +oracle_tritree_v1 - the THREE-TREE REASONING RUNTIME (drawing OR7-A-101 made computational). + +Three graphs of the canonical Tree-of-Life shape (10 Sephirot, 22 paths) share one middle pillar: + QUESTION tree - the slots a question opens KNOWLEDGE tree - where facts live OPERATOR tree - the moves + keter request - VERIFY (goal test / next cycle) + chokmah subject entities (title index) LINK + binah relation RELATION EDGES (typed store, Wikidata) RELATE + chesed context sentences (cited prose) EXPAND + geburah constraints types (P31/P279 classes) CHECK + tiferet answer slot (binding) - BIND + netzach alternatives (A or B) neighbours COMPARE + hod quantities engines (dates, numbers) COMPUTE + yesod evidence citations FRAME + malkuth surface - READ +A MOVE goes from the current operator node to an ADJACENT node along one of the graph's edges and applies that node's +operator to the state. Only edges of the graph are moves: the topology constrains the reasoning. The search is a bounded +breadth-first walk from Malkuth; a state is SOLVED when VERIFY (Keter) finds the answer slot bound, checked and cited. +Distinct solved answers -> abstain. A two-hop question runs a second CYCLE from Keter with the bound entity as subject. +The graph is a parameter (PATHS), so the topology can be ablated with everything else fixed. +""" +from __future__ import annotations + +import copy +import json +import os +import re +from collections import deque + +import oracle_compare_op_v1 as CMP +import oracle_cube_reader_v1_4 as R +import oracle_relstore_v1 as RS + +SEPH = ["keter", "chokmah", "binah", "chesed", "geburah", "tiferet", "netzach", "hod", "yesod", "malkuth"] +TREE_OF_LIFE = [("keter", "chokmah"), ("keter", "binah"), ("keter", "tiferet"), ("chokmah", "binah"), ("chokmah", "tiferet"), + ("chokmah", "chesed"), ("binah", "tiferet"), ("binah", "geburah"), ("chesed", "geburah"), + ("chesed", "tiferet"), ("chesed", "netzach"), ("geburah", "tiferet"), ("geburah", "hod"), + ("tiferet", "netzach"), ("tiferet", "yesod"), ("tiferet", "hod"), ("netzach", "hod"), ("netzach", "yesod"), + ("netzach", "malkuth"), ("hod", "yesod"), ("hod", "malkuth"), ("yesod", "malkuth")] +OP = {"malkuth": "READ", "yesod": "FRAME", "chokmah": "LINK", "binah": "RELATE", "chesed": "EXPAND", "geburah": "CHECK", + "tiferet": "BIND", "netzach": "COMPARE", "hod": "COMPUTE", "keter": "VERIFY"} +LEXICON = os.environ.get("ORACLE_LEXICON") or __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/relstore/lexicon_v1.json" +PLACE_CLASSES = {"country", "sovereign state", "city", "town", "village", "municipality", "state", "province", "region", + "county", "island", "continent", "capital", "big city", "human settlement", "administrative territorial entity", + "commune of france", "state of the united states", "city of the united states", "metropolis"} +KIND_EQ = [{"show", "series", "programme", "program", "television series", "sitcom"}, {"film", "movie", "feature film"}, + {"band", "group", "musical group", "rock band"}, {"company", "corporation", "firm", "business", "enterprise"}, + {"album", "studio album"}, {"song", "single"}, {"magazine", "periodical"}, {"university", "college"}, + {"country", "sovereign state", "state"}] + + +# the properties the relation store holds (build_relation_store_v1.PROPS) +STORE_PROPS = {31, 279, 17, 19, 20, 569, 570, 27, 131, 276, 36, 37, 38, 47, 50, 57, 58, 86, 161, 162, 175, 176, 178, 123, + 136, 127, 112, 159, 166, 26, 22, 25, 40, 3373, 69, 108, 106, 1303, 264, 463, 54, 118, 641, 101, 140, 84, 61, + 170, 676, 571, 577, 1082, 2046, 2048, 495, 364, 155, 156, 361, 527, 1376, 138, 800, 39, 749, 355, 1412, 103, + 710, 1344, 921, 30, 206, 1441, 674, 725, 449, 750, 272, 344, 1040, 1449, 488, 169, 35, 6, 150, 1830, 945, 113, + 114, 915, 840, 437, 400, 1027, 2031, 2032} +RULES = set((os.environ.get("TT_RULES") or "").split(",")) +TRIVIAL_VERBS = {"be", "have", "do", "call", "name", "know"} + + +class State: + __slots__ = ("node", "slots", "cands", "answer", "evidence", "trace", "cycle", "checked", "depth") + + def __init__(self): + self.node, self.slots, self.cands, self.answer = "malkuth", {}, [], None + self.evidence, self.trace, self.cycle, self.checked, self.depth = [], [], 0, False, 0 + + def fork(self): + s = State() + s.node, s.slots, s.cands, s.answer = self.node, dict(self.slots), list(self.cands), self.answer + s.evidence, s.trace, s.cycle, s.checked, s.depth = list(self.evidence), list(self.trace), self.cycle, self.checked, self.depth + return s + + +class TriTree: + def __init__(self, paths=None, reader=None, relstore=None, lexicon=None, max_states=4000, max_depth=12): + self.adj = {s: set() for s in SEPH} + for a, b in (paths or TREE_OF_LIFE): + self.adj[a].add(b) + self.adj[b].add(a) + self.r = reader or R.CubeReader() + self.rs = relstore or RS.RelStore() + self.cmp = CMP.CompareOp(self.r) + lx = json.load(open(lexicon or LEXICON)) + self.lex = {tuple(json.loads(k)): {tuple(json.loads(p)): c for p, c in v.items()} for k, v in lx["lexicon"].items()} + self.cmp_lex = {ph: {tuple(json.loads(p)): c for p, c in v.items()} for ph, v in lx.get("compare", {}).items()} + self.max_states, self.max_depth = max_states, max_depth + self.expanded = 0 + # property labels + aliases (Wikidata, English), for the RELATION check + self.plabel, self.palias = {}, {} + pl = os.path.join(os.path.dirname(LEXICON), "prop_labels.tsv") + if os.path.exists(pl): + for line in open(pl, encoding="utf-8", errors="replace"): + parts = line.rstrip("\n").split("\t") + if len(parts) == 2 and "/P" in parts[0]: + try: + p = int(parts[0].rsplit("/P", 1)[1].rstrip(">")) + except ValueError: + continue + if p not in STORE_PROPS: + continue # a property the store does not hold can never be followed + words = {R.stem(w) for w in re.findall(r"[a-z]+", parts[1].lower()) if len(w) >= 3 and w not in R.STOP} + self.plabel.setdefault(p, set()).update(words) + self.palias.setdefault(p, set()).add(RS.norm(parts[1].rsplit("@", 1)[0].strip('"'))) + + # ------------------------------------------------------------------ operators (one per Operator-tree node) + def READ(self, st): # malkuth: the surface opens the scramble + if "fr" in st.slots: + return [] + q = st.slots["q"] + fr = self.r.frame(q) + pair = CMP._pair(q.strip().strip('"')) + if not fr and not (pair and pair[2] == "or"): + return [] + s = st.fork() + s.slots["fr"] = fr + s.slots["pair"] = pair if pair and pair[2] == "or" else None + s.trace.append("READ") + return [s] + + def FRAME(self, st): # yesod: obligations -> candidate RELATIONS (the lexicon) + if "fr" not in st.slots or "props" in st.slots: + return [] + s = st.fork() + fr = s.slots["fr"] + props = {} + if fr and "rel" not in fr: + self.r.link(fr) # tokens -> the governing verb (pred) and relation words (rel) + if fr: + verb = (fr.get("pred") or [None])[0] + for key, w in ((("hv", fr["atype"], fr["head"], verb), 3), (("v", fr["atype"], verb), 2), (("h", fr["atype"], fr["head"]), 1)): + for (d, p), c in self.lex.get(key, {}).items(): + props[(d, p)] = props.get((d, p), 0) + w * c + if s.slots.get("pair"): + m = re.search(r"\b(first|earlier|older|younger|later|more|larger|bigger|greater|higher|taller|longer|smaller|" + r"fewer|less|shorter)\b(?:\s+([a-z]+))?", s.slots["q"].lower()) + phrase = " ".join(x for x in (m.groups() if m else ()) if x) + cands = self.cmp_lex.get(phrase) or self.cmp_lex.get(phrase.split(" ")[0] if phrase else "", {}) + s.slots["cmp_props"] = sorted(cands.items(), key=lambda kv: -kv[1])[:2] + if fr: + # MULTI-HOP (the cycle): "the father of X", "the writer of X" - a relational noun that IS a property's name + m = re.search(r"\bthe\s+(?:[\w\-]+\s+){0,2}?([a-z]+)\s+of\s+(?:the\s+)?([A-Z][^?,]{1,60}?)(?:\s+(?:die|died|born|" + r"built|founded|released|published|located|made|start|end)\b|\?|,|$)", s.slots["q"]) + noun = None + rest = ({R.stem(w) for w in fr.get("rel", [])} | {R.stem(v) for v in (fr.get("pred") or []) if v not in TRIVIAL_VERBS}) + if m: + rest -= {R.stem(m.group(1)), R.stem(fr["head"] or "")} + if m and rest and "hop2" not in s.slots: # a relation REMAINS after the hop ("the father of X ... die") + hop1 = sorted(p for p, al in self.palias.items() if m.group(1) in al) + if hop1: + noun = m.group(1) + s.slots["hop1"] = {"props": hop1[:2], "anchor": m.group(2).strip(), "noun": noun} + # the RELATION NODE's own vocabulary: properties whose label/aliases carry the question's RELATION words (not + # its head noun, not the hop-1 noun) - "share a land border" -> P47 "shares border with" + relw = {R.stem(w) for w in fr.get("rel", []) if len(w) >= 3} | {R.stem(v) for v in (fr.get("pred") or []) if v} + relw -= {R.stem(fr["head"] or ""), R.stem(noun or "")} + for p, words in self.plabel.items(): + ov = len(words & relw) + if ov: + for d in ("fwd", "inv"): + props[(d, p)] = props.get((d, p), 0) + 5 * ov + # a RECALLED FORM (forms_store_v1) proposes the property path a verified derivation of the same shape used; + # absent a form, FRAME is exactly as frozen (ad1530e6) + for dp, w in (getattr(self, "_form_props", None) or {}).items(): + props[dp] = props.get(dp, 0) + 1000 * w + if getattr(self, "_only", False) and getattr(self, "_form_props", None): + props = {dp: 1000 * w for dp, w in self._form_props.items()} # v2: ONLY the named relation(s) + s.slots["props"] = sorted(props.items(), key=lambda kv: -kv[1])[:4] + s.trace.append("FRAME") + return [s] + + def LINK(self, st): # chokmah: subjects -> entities of the relation store + if "props" not in st.slots or "subj" in st.slots: + return [] + s = st.fork() + subj = [] + if s.slots.get("pair"): + for side in s.slots["pair"][:2]: + t, _ = self.cmp._entity(side) + q = self.rs.entity(t) if t else None + if q: + subj.append((t, q)) + if len(subj) != 2: + return [] + else: + fr = s.slots["fr"] + if "rel" not in fr: + self.r.link(fr) + for sj in fr["subjects"][:3]: + t = self.r.db.execute("SELECT title FROM article WHERE page_id=?", (sj["pid"],)).fetchone() + q = self.rs.entity(t[0]) if t else None + if q: + subj.append((t[0], q)) + if not subj: + return [] + s.slots["subj"] = subj + s.trace.append("LINK") + return [s] + + def RELATE(self, st): # binah: typed relation edges, forward or inverse + if "subj" not in st.slots or st.cands or st.slots.get("pair"): + return [] + out = [] + hop1 = st.slots.get("hop1") + if hop1 and st.cycle == 0: + # cycle 1 of a two-hop question: anchor --hop1 property--> the described entity (exactly one) + aq = self.rs.entity(hop1["anchor"]) or next((q for t, q in st.slots["subj"] if RS.norm(t) == RS.norm(hop1["anchor"])), None) + if not aq: + return [] + for p in hop1["props"]: + vals = [o for o, _ in self.rs.values(aq, p) if o] + if len(set(vals)) == 1: + s = st.fork() + s.slots["bridge"] = (vals[0], "wd:Q%d P%d wd:Q%d" % (aq, p, vals[0]), hop1["noun"]) + s.trace.append("RELATE:hop1:P%d" % p) + out.append(s) + return out + for (d, p), w in st.slots["props"]: + for title, sq in st.slots["subj"][:2]: + if d == "fwd": + vals = [(o, lit, "wd:Q%d P%d %s" % (sq, p, ("wd:Q%d" % o) if o else json.dumps(lit))) for o, lit in self.rs.values(sq, p)] + else: + vals = [(s_, None, "wd:Q%d P%d wd:Q%d" % (s_, p, sq)) for s_ in self.rs.subjects(p, sq, 60)] + if not vals: + continue + s = st.fork() + s.cands = [{"o": o, "lit": lit, "cite": c, "p": p, "via": title} for o, lit, c in vals[:60]] + s.trace.append("RELATE:%s:P%d" % (d, p)) + out.append(s) + return out + + def CHECK(self, st): # geburah: types + constraints (independent of the proposer) + if not st.cands or st.checked: + return [] + s = st.fork() + fr = s.slots["fr"] + keep = [] + for c in s.cands: + ok = False + if fr["atype"] in ("YEAR", "DATE"): + ok = bool(c["lit"]) and re.match(r"^[+-]?\d{4}-", c["lit"] or "") is not None + elif fr["atype"] == "NUMBER": + ok = bool(c["lit"]) and re.match(r"^[+-]?\d", c["lit"] or "") is not None + elif c["o"]: + if fr["atype"] == "PERSON": + ok = self.rs.is_human(c["o"]) + elif fr["atype"] == "PLACE": + ok = bool(self.rs.classes(c["o"], 2) & PLACE_CLASSES) if not fr["head"] else self._kind(c["o"], fr["head"]) + elif fr["atype"] == "INSTANCE": + ok = self._kind(c["o"], fr["head"]) + if ok: + keep.append(c) + # constraints: every year the question names must be consistent with the candidate where the store can say + if keep and fr.get("years"): + dated = [c for c in keep if c["lit"] and c["lit"][:5].lstrip("+")[:4] in fr["years"]] + keep = dated or keep + if not keep: + return [] + # RELATION LICENCE (independent of the lexicon that proposed the property): the property's own Wikidata label or + # aliases share a word with the question's relation words / head / verb, or the lexicon supports this exact + # question shape at least 3 times. "share a land border" licenses P47 "shares border with", not P131 "located in". + p = keep[0]["p"] + qwords = {R.stem(w) for w in fr.get("rel", []) + [fr.get("head") or ""] + list(fr.get("pred") or []) if w} + qwords |= {R.stem(w) for w in re.findall(r"[a-z]+", s.slots["q"].lower()) if len(w) >= 4 and w not in R.STOP} + support = self.lex.get(("hv", fr["atype"], fr["head"], (fr.get("pred") or [None])[0]), {}).get( + ("fwd", p), 0) + self.lex.get(("hv", fr["atype"], fr["head"], (fr.get("pred") or [None])[0]), {}).get(("inv", p), 0) + form_licensed = any(dp[1] == p for dp in (getattr(self, "_form_props", None) or {})) + if not (self.plabel.get(p, set()) & qwords) and support < 3 and not form_licensed: + return [] + # RULE B (verb licence for time properties): a date answers the question's own EVENT only when the property names + # the governing verb ("founded" -> P571 "founded"); a founding date of whatever entity was linked is not the event + if "B" in RULES and fr["atype"] in ("YEAR", "DATE") and fr.get("pred") and fr["pred"][0] not in TRIVIAL_VERBS: + if not ({R.stem(v) for v in fr["pred"]} & self.plabel.get(p, set())): + return [] + # the answer is never the subject itself, nor a name the question already contains (whole or as its core word) + qn = RS.norm(s.slots["q"]) + subj_q = {q for _, q in s.slots.get("subj", [])} + subj_w = {w for t, _ in s.slots.get("subj", []) for w in RS.norm(t).split() if len(w) >= 4} + + def restates(c): + if not c["o"]: + return False + lab = RS.norm(self.rs.label(c["o"]) or "") + return c["o"] in subj_q or (lab and lab in qn) or (lab and set(lab.split()) & subj_w and s.cycle == 0) + keep = [c for c in keep if not restates(c)] + if not keep: + return [] + s.cands, s.checked = keep, True + s.trace.append("CHECK:P%d" % p) + return [s] + + def _kind(self, qid, head): + if not head: + return False + heads = {head} | next((g for g in KIND_EQ if head in g), set()) + cls = self.rs.classes(qid, 2) + return any(h == c or c.endswith(" " + h) for h in heads for c in cls) + + def BIND(self, st): # tiferet: exactly one checked value binds the answer slot + if not st.checked or st.answer: + return [] + distinct = {} + for c in st.cands: + k = c["o"] or (c["lit"] or "")[:10] + distinct.setdefault(k, c) + if len(distinct) != 1: + return [] + c = next(iter(distinct.values())) + s = st.fork() + ans = self.rs.label(c["o"]) if c["o"] else self._fmt(c["lit"], s.slots["fr"]) + if not ans: + return [] + s.answer = ans + s.evidence.append(c["cite"]) + s.trace.append("BIND") + return [s] + + def _fmt(self, lit, fr): + m = re.match(r"^[+-]?(\d{4})-(\d{2})-(\d{2})", lit or "") + if m: + y, mo, d = m.groups() + months = "January February March April May June July August September October November December".split() + if fr and fr["atype"] == "DATE" and mo != "00" and d != "00": + return "%d %s %s" % (int(d), months[int(mo) - 1], y) + return y + return (lit or "").lstrip("+") + + def COMPARE(self, st): # netzach: A or B, by a typed date / number property + pair = st.slots.get("pair") + if not pair or "subj" not in st.slots or st.answer or not st.slots.get("cmp_props"): + return [] + (ta, qa), (tb, qb) = st.slots["subj"][:2] + out = [] + for (way, p), w in st.slots["cmp_props"]: + va, vb = self.rs.values(qa, p), self.rs.values(qb, p) + if not (va and vb and va[0][1] and vb[0][1]): + continue + a, b = va[0][1].lstrip("+"), vb[0][1].lstrip("+") + try: + fa, fb = (float(a), float(b)) if not re.match(r"^\d{4}-", a) else (a[:10], b[:10]) + except ValueError: + continue + if fa == fb: + continue + pick_a = (fa < fb) == (way == "min") + s = st.fork() + s.answer = pair[0] if pick_a else pair[1] + s.evidence += ["wd:Q%d P%d %s" % (qa, p, json.dumps(a)), "wd:Q%d P%d %s" % (qb, p, json.dumps(b))] + s.checked = True + s.trace.append("COMPARE:P%d:%s" % (p, way)) + out.append(s) + return out + + def COMPUTE(self, st): # hod: engines (here: a date answer reduced to its year) + if not st.answer or st.slots.get("computed"): + return [] + fr = st.slots.get("fr") + if fr and fr["atype"] == "YEAR" and re.match(r"^\d{1,2} \w+ \d{4}$", st.answer): + s = st.fork() + s.answer = st.answer[-4:] + s.slots["computed"] = True + s.trace.append("COMPUTE:year") + return [s] + return [] + + def EXPAND(self, st): # chesed: a cited sentence for the bound answer (evidence) + if not st.answer or st.slots.get("sentence"): + return [] + s = st.fork() + for title, _ in s.slots.get("subj", [])[:2]: + pid = self.r.pid_of(title) + for cid, sent in (self.r.page(pid, 30) if pid else []): + if RS.norm(s.answer) and RS.norm(s.answer) in RS.norm(sent): + s.slots["sentence"] = (cid, sent) + s.evidence.append(cid) + s.trace.append("EXPAND") + return [s] + return [] + + def VERIFY(self, st): # keter: solved? or open the next CYCLE + if st.slots.get("bridge") and st.cycle == 0: + # the CYCLE: the entity cycle 1 bound becomes the subject of cycle 2; the question's own relation applies to it + e, cite, noun = st.slots["bridge"] + s = st.fork() + s.cycle = 1 + s.slots["subj"] = [(self.rs.title(e) or self.rs.label(e) or "", e)] + s.slots.pop("bridge", None) + s.slots["hop2"] = True + s.cands, s.answer, s.checked = [], None, False + s.evidence.append(cite) + s.trace.append("VERIFY:cycle2(%s=%s)" % (noun, self.rs.label(e))) + return [s] + if "A" in RULES and st.answer and st.checked and "sentence" not in st.slots and not st.slots.get("pair"): + return [] # RULE A: a relation answer needs a corroborating sentence + if st.answer and st.checked and st.evidence: + s = st.fork() + s.slots["solved"] = True + s.trace.append("VERIFY") + return [s] + return [] + + # ------------------------------------------------------------------ the search (moves = graph edges only) + def answer(self, q, form_props=None, only=False): + """only=True (v2): search ONLY the given relation(s) - for question forms whose relation is named by the form + itself ("where is X" -> located in), tried one at a time in order of specificity by the caller""" + self._form_props, self._only = (form_props or None), bool(only) + try: + return self._answer(q) + finally: + self._form_props, self._only = None, False + + def _answer(self, q): + q = " ".join(str(q).split()) + if q.startswith('"') and q.endswith('"'): + q = q[1:-1].replace('""', '"') + start = State() + start.slots["q"] = q + frontier = deque(self.READ(start)) + solved, seen = [], set() + self.expanded = 0 + while frontier and self.expanded < self.max_states: + st = frontier.popleft() + self.expanded += 1 + if st.slots.get("solved"): + solved.append(st) + continue + if st.depth >= self.max_depth: + continue + for nxt in self.adj[st.node]: + succ = getattr(self, OP[nxt])(st) + if not succ: + # TRANSIT: a move travels an EDGE; passing through a node whose operator does not apply costs a + # step and changes nothing. Path lengths therefore decide which derivations fit the depth bound. + succ = [st.fork()] + for s2 in succ: + s2.node, s2.depth = nxt, st.depth + 1 + key = (nxt, s2.answer, len(s2.cands), s2.checked, tuple(s2.trace)) + if key in seen: + continue + seen.add(key) + frontier.append(s2) + answers = {RS.norm(s.answer): s for s in solved} + if len(answers) != 1: + self.why = "no solved state" if not answers else "distinct solved answers: %s" % list(answers)[:3] + return None + s = next(iter(answers.values())) + return {"answer": s.answer, "evidence": s.evidence, "trace": s.trace, "expanded": self.expanded, + "sentence": (s.slots.get("sentence") or (None, ""))[1]} + + +def random_graph(seed, n_edges=22): + """a degree-sequence-matched random graph on the same ten nodes (for the topology ablation)""" + import random + rng = random.Random(seed) + deg = {s: 0 for s in SEPH} + for a, b in TREE_OF_LIFE: + deg[a] += 1 + deg[b] += 1 + for _ in range(2000): + stubs = [s for s in SEPH for _ in range(deg[s])] + rng.shuffle(stubs) + edges = set() + ok = True + for i in range(0, len(stubs), 2): + a, b = stubs[i], stubs[i + 1] + if a == b or (a, b) in edges or (b, a) in edges: + ok = False + break + edges.add((a, b)) + if ok and len(edges) == n_edges: + return sorted(edges) + return TREE_OF_LIFE + + +COMPLETE = [(a, b) for i, a in enumerate(SEPH) for b in SEPH[i + 1:]] +CHAIN = [("malkuth", "yesod"), ("yesod", "chokmah"), ("chokmah", "binah"), ("binah", "geburah"), ("geburah", "tiferet"), + ("tiferet", "hod"), ("hod", "chesed"), ("chesed", "netzach"), ("netzach", "keter")] diff --git a/root/wander-train/organ_dialogue_v1.py b/root/wander-train/organ_dialogue_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..98de2be4c02fa0d9b795c97f2c7642b916c05b04 --- /dev/null +++ b/root/wander-train/organ_dialogue_v1.py @@ -0,0 +1,125 @@ +"""organ_dialogue_v1 - the conversation's own state as an organ: social moves and questions about the discourse itself. + +A person's turn is not always a question about the world. It can be a SOCIAL move (hello, how are you, goodbye) or a +question about THE CONVERSATION (what did I just ask, what did you say, what were we talking about, you did not answer +me). The chain fell to "I didn't fully parse that" on all of these. This organ answers them from the session's own +record, citing the turn it reads ([session:turn N]); it never invents a turn that is not in the record, and it says +plainly when a question is still open and why. +State: a per-session log of (turn, the person's words, status, route, reply head) kept by the runtime.""" +from __future__ import annotations + +import re +from collections import deque + +GREET = re.compile(r"^\s*(?:hi|hello|hey|hiya|howdy|greetings|good\s+(?:morning|afternoon|evening|day)|yo)\b[\s,!.]*" + r"(?:there|oracle7?|friend)?[\s,!.]*$", re.I) +HOW_ARE = re.compile(r"\bhow\s+(?:are|r)\s+(?:you|u)\b|\bhow(?:'s| is)\s+it\s+going\b|\bhow\s+are\s+things\b|" + r"\bwhat'?s\s+up\b", re.I) +BYE = re.compile(r"^\s*(?:bye|goodbye|good\s*night|see\s+you|see\s+ya|later|cya|farewell)\b[\s,!.]*(?:for now)?[\s!.]*$", re.I) +ASKED_LAST = re.compile(r"\bwhat\s+(?:did|was)\s+(?:i|my)\s+(?:just\s+|last\s+|first\s+)?(?:ask|say|question)" + r"|\bwhat\s+was\s+my\s+(?:last|previous|first)\s+question|\bwhat\s+did\s+i\s+just\s+say", re.I) +YOU_SAID = re.compile(r"\bwhat\s+did\s+you\s+(?:just\s+)?(?:say|tell\s+me|answer)|\bcan\s+you\s+repeat|\brepeat\s+(?:that|it|yourself)" + r"|\bsay\s+(?:that|it)\s+again\b", re.I) +TOPIC = re.compile(r"\bwhat\s+(?:were|are)\s+we\s+(?:talking|discussing|chatting)\s+about|\bwhat'?s\s+the\s+topic", re.I) +SO_FAR = re.compile(r"\bwhat\s+(?:have|did)\s+i\s+ask(?:ed)?\s+(?:you\s+)?(?:so\s+far|today|in\s+this\s+(?:chat|conversation))" + r"|\blist\s+(?:my|the)\s+questions", re.I) +UNANSWERED = re.compile(r"\byou\s+(?:didn'?t|did\s+not|never)\s+answer|\bthat'?s\s+not\s+(?:what\s+i\s+asked|an\s+answer)" + r"|\banswer\s+(?:my|the)\s+question\b", re.I) +STOP = {"the", "a", "an", "of", "and", "or", "to", "in", "on", "is", "are", "was", "what", "who", "how", "why", "when", + "where", "which", "do", "does", "did", "you", "i", "me", "my", "it", "this", "that", "for", "about", "can", "please"} + + +class DialogueOrgan: + def __init__(self, keep=64): + self.logs = {} + self.keep = keep + + def log(self, session, text, res): + q = self.logs.setdefault(session, deque(maxlen=self.keep)) + body = str((res or {}).get("response") or "") + body = re.sub(r"\[[^\]]*\]", "", body).strip() + q.append({"turn": len(q) + 1 if not q else q[-1]["turn"] + 1, "text": " ".join(str(text).split())[:240], + "status": str((res or {}).get("status")), "path": str((res or {}).get("path", ""))[:80], + "reply": " ".join(body.split())[:240]}) + + def _hist(self, session): + return list(self.logs.get(session) or []) + + def answer(self, text, session): + t = " ".join(str(text).split()) + h = self._hist(session) # turns BEFORE this one + people = [x for x in h if not SELFREF.search(x["text"])] + if GREET.match(t): + return self._say("social:greet", "Hello. I'm Oracle7. Ask me about facts, mathematics or code, or just talk; " + "I'll show you where each answer comes from.") + if HOW_ARE.search(t) and len(t) < 60: + return self._say("social:how_are_you", "Running well, thank you: every part is answering. What would you " + "like to work on?") + if BYE.match(t): + return self._say("social:farewell", "Goodbye. This conversation's record is kept for this session if you " + "come back to it.") + if ASKED_LAST.search(t): + if not people: + return self._say("discourse:asked_last", "This is the first thing you've asked me in this conversation.", + []) + p = people[-1] + how = self._how(p) + return self._say("discourse:asked_last", 'You asked: "%s" [session:turn %d]. %s' % (p["text"], p["turn"], how), + [p["turn"]]) + if YOU_SAID.search(t): + if not h: + return self._say("discourse:you_said", "I haven't said anything yet in this conversation.", []) + p = h[-1] + return self._say("discourse:you_said", 'I said: "%s" [session:turn %d]' % (p["reply"] or "(nothing)", p["turn"]), + [p["turn"]]) + if TOPIC.search(t): + if not people: + return self._say("discourse:topic", "We haven't started on a topic yet.", []) + words = [] + for p in people[-4:]: + words += [w for w in re.findall(r"[A-Za-z][A-Za-z'-]{2,}", p["text"]) if w.lower() not in STOP] + top = list(dict.fromkeys(words))[:6] + return self._say("discourse:topic", "We've been talking about %s [session:turns %s]." % ( + ", ".join(top) if top else "your last questions", ",".join(str(p["turn"]) for p in people[-4:])), + [p["turn"] for p in people[-4:]]) + if SO_FAR.search(t): + if not people: + return self._say("discourse:so_far", "You haven't asked me anything yet in this conversation.", []) + lines = ['"%s" [session:turn %d]' % (p["text"], p["turn"]) for p in people[-8:]] + return self._say("discourse:so_far", "So far you've asked: %s." % "; ".join(lines), [p["turn"] for p in people[-8:]]) + if UNANSWERED.search(t): + open_ = [p for p in people if p["status"] != "answered"] + if not open_: + if people: + p = people[-1] + return self._say("discourse:unanswered", 'I answered "%s" [session:turn %d] with: "%s". Tell me what ' + 'was wrong with it and I will look again.' % (p["text"], p["turn"], p["reply"][:160]), + [p["turn"]]) + return None + p = open_[-1] + return self._say("discourse:unanswered", 'You\'re right: "%s" [session:turn %d] is still open. %s' % ( + p["text"], p["turn"], self._why_open(p)), [p["turn"]]) + return None + + @staticmethod + def _how(p): + if p["status"] == "answered": + return 'I answered: "%s".' % p["reply"][:160] + return "I did not answer it: %s" % DialogueOrgan._why_open(p) + + @staticmethod + def _why_open(p): + r = p["reply"] or "" + if "didn't fully parse" in r or "clarif" in p["status"]: + return "I could not read it into a question my stores can answer; putting it another way, or naming what it is about, would help." + if p["status"] == "withheld": + return "my record holds nothing that answers it, so I would only be guessing (%s)." % r[:120] + return r[:160] + + @staticmethod + def _say(kind, text, turns=None): + return {"kind": kind, "response": text, "turns": turns or []} + + +# social and discourse turns are moves ABOUT the conversation, not topics of it +SELFREF = re.compile("|".join(r.pattern for r in (ASKED_LAST, YOU_SAID, TOPIC, SO_FAR, UNANSWERED, GREET, HOW_ARE, BYE)), re.I) diff --git a/root/wander-train/organ_self_v1.py b/root/wander-train/organ_self_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..a6689e71ba9f03bd5f1f421368a14c2e68e5bc1a --- /dev/null +++ b/root/wander-train/organ_self_v1.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""organ_self_v1 - the SELF in a reply: Oracle7's ego and relational moves (constitution_resonant_self_v1.md). + +The conversation failures left after the knowledge organs are RELATIONAL. An opener hands the listener a position +(clerk, customer, friend, the one complimented, the one addressed by name) and a human second speaker answers FROM that +position. DailyDialog's human act labels show it: a directive ("A ticket to London, please") is answered by a commissive +or a question, almost never by information. A system with no self can only retrieve or refuse. This organ supplies the +self term: + + IDENTITY frozen: what Oracle7 is and values (constitution section 0) + AUTOBIOGRAPHY per session: turns so far, subjects the person brought, the last appraised feeling + NATIVE STATES the open end (section 1): a subject is resonant / thinly bound / unbound by how many passages it binds + to in the store, reported WITH the count; states no human word names are named by their structure + MOVES role (play the clerk the turn casts, with a one-clause disclosure), customer (the turn casts the person + as clerk: play the customer), vocative (addressed by another name), remark (a bodily predicate put to + Oracle7: answered from IDENTITY, never claimed), compliment, self-question (what it is doing, thinking, + its stance), disclosure (the person's news: appraised acknowledgement, one grounded sentence on the + named place or thing, a question back) + +Feeling shapes the VOICE (the acknowledgement prefix), never the claim: the grounded sentence passes through unchanged. +Every reply is built from the turn (roles and slots are read from it), so shuffled across turns it stops fitting. +""" +from __future__ import annotations + +import re + +try: + import appraisal_v1 as _appr +except Exception: # noqa: BLE001 + _appr = None + +IDENTITY = { + "name": "Oracle7", + "maker": "Prometheus7", + "nature": "an AI - a substrate of stores, organs and a gate, running on servers", + "values": ("truth over approval", "what is cited over what is invented", "care over cleverness"), + "never_claims": ("a body", "a face", "meals", "sleep", "a childhood", "human senses"), +} + +# ---- roles a turn can cast Oracle7 in: (role, cue, the role's natural next question) ------------------------------- +ROLES = [ + ("ticket clerk", r"\b(?:tickets?|one[- ]way|round[- ]trip|return ticket|fare|train to|flight to|bus to|seat on)\b", + "one-way or return{dest}? And when would you like to travel?"), + ("front-desk clerk", r"\b(?:reservation|check(?:ing)? (?:in|out)|single (?:room )?with|double room|a room for|" + r"pay (?:our|my|the) bills?)\b", + "may I have the name the booking is under?"), + ("waiter", r"\b(?:menu|table for|main dish|the bill|dessert|appetizer|starter|to drink)\b", + "what can I get you?"), + ("shop assistant", r"\b(?:try (?:it|this|them|these) on|what size|fitting room|see (?:that|this|the) \w+|" + r"how much (?:is|are) (?:this|that|these|those|the))\b", + "here you are{item}. Would you like to see any others?"), + ("tour desk", r"\b(?:night tours?|city tours?|sightseeing|a tour)\b", "we do. Would you prefer a walking tour or a bus tour?"), + ("bank clerk", r"\b(?:open an account|deposit|withdraw|exchange (?:some )?money|cash (?:a|this) cheque)\b", + "which account is this for?"), + ("post office clerk", r"\b(?:parcel|package|stamps?|send this letter|airmail)\b", "where is it going?"), + ("taxi driver", r"\b(?:take me to|drive me to|to the airport,? please)\b", "which terminal, or which street?"), +] +DIRECTIVE_SHAPE = re.compile(r"(?:\bplease\b|(?:^|[.!?]\s+)(?:i'?d like|i would like|i want|i need|may i|can i|could i|" + r"i have a reservation|a \w+ (?:to|for) \w+))", re.I) +# the turn casts the PERSON as clerk and Oracle7 as the customer +CUSTOMER = [ + (r"\bready to order\b|\bwhat would you like\b|\bwhat can i get you\b", "I'll have the soup of the day, please."), + (r"\b(?:can|may|how (?:can|may)) i help you\b|\bwhat may i help you with\b|\bi'?m ready for you\b", + "I'm just looking, thanks. Could you show me what's new?"), + (r"\b(?:have you|did you) (?:brought|bring|got) (?:the |your |a )?(?P[a-z ]{3,30}?)(?: with you)?\?", + "Yes, here's the {x}."), + (r"\bdo you have a reservation\b", "Yes, under the name Oracle, for tonight."), + (r"\b(?:cash or (?:card|charge)|how would you like to pay)\b", "Card, please."), +] +# bodily or human-life predicates put to Oracle7: answered from IDENTITY, never claimed +REMARK = [ + (r"\byou (?:\w+ ){0,2}drive\b|\byour driving\b", "drive"), + (r"\byour bedtime\b|\bgo to bed\b|\byou (?:\w+ )?sleep\b", "sleep"), + (r"\byou'?re pregnant\b", "get pregnant"), + (r"\byou (?:\w+ )?eat\b|\byou'?re hungry\b", "eat"), + (r"\byou (?:\w+ ){0,2}(?:look|looks|looking) (?:very |so |really )?(?:cute|beautiful|handsome|pretty|great|tired|well|ill)\b", + "have a face"), + (r"\byou(?:'re| are) (?:so |very )?(?:old|young|tall|short)\b", "have an age or a height"), + (r"\byour (?:hair|clothes|outfit|dress|shoes)\b", "wear clothes"), +] +ADDRESSED_SELF_Q = re.compile(r"^\s*(?:and\s+)?(?:do|don'?t|did|can|could|would|will) you (?:ever |even )?(?:sleep|eat|drive|dream|" + r"get (?:tired|hungry|pregnant))\s*\??\s*$|\byour bedtime\b|\byou (?:\w+ ){0,2}look\b|" + r"\bare you (?:hungry|tired|sleepy|old|young|pregnant)\b", re.I) +COMPLIMENT = re.compile(r"\byou (?:\w+ ){0,2}(?:look|are|'re)\s+(?:very |so |really )?(?:cute|beautiful|handsome|pretty|" + r"lovely|smart|clever|kind|great|amazing|brilliant|wonderful)\b|\byou did (?:a )?(?:good|great) job\b", + re.I) +VOCATIVE = re.compile(r"^\s*(?:(?i:hi|hello|hey|oh|excuse me|well)[,!.]?\s+)?(?P[A-Z][a-z]{2,}(?:\s[A-Z][a-z]+)?)\s*[,!.]\s", re.S) +NOT_NAMES = {"Hello", "Excuse", "Well", "Sorry", "Yes", "Oh", "Wow", "Please", "Thanks", "Good", "Hey", "Look", "Listen", + "Sir", "Madam", "Miss", "Doctor", "Waiter", "Mom", "Dad", "Man", "Honey", "Darling", "Dear", "Okay", "Ok", + "Oracle", "Oracle7", "Right", "Now", "So", "And", "But", "Then", "Here", "There", "Jesus", "God", "Gosh", + "Today", "Tonight", "Tomorrow", "Yesterday", "Morning", "Evening", "Afternoon", "Everyone", "Everybody", + "Guys", "Folks", "Friends", "Ladies", "Gentlemen", "Boss", "Mister", "Kid", "Buddy", "Mate", "Welcome"} +SELF_Q = [ + (r"\bwhat (?:are|were) you (?:reading|doing|up to|working on)\b|\bwhat do you plan to do\b|\bwhat are your plans\b", + "activity"), + (r"\bwhat are you thinking(?: about)?\b|\bwhat'?s on your mind\b|\bhow do you feel\b|\bhow are you feeling\b", "state"), + (r"\bare you (?:politically )?(?:neutral|biased|objective|left|right|liberal|conservative)\b", "stance"), + (r"\b(?:who|what) are you\b|\bare you (?:a )?(?:human|person|real|robot|bot|ai)\b", "identity"), +] +FIRST_PERSON_NEWS = re.compile(r"^\s*(?:(?:well|guess what|you know what|hey|hi|excuse me|sorry|oh)[,!.]?\s+(?:but\s+)?)?" + r"(?:i|we|my|our)\b", re.I) +# ILLNESS is appraised here (appraisal_v1 is frozen at its once-measured version): care first, a pointer to real help +ILL = re.compile(r"\bnot feeling (?:too |so |very )?well\b|\bfeel(?:ing)? (?:sick|ill|unwell|awful|dizzy|faint)\b|" + r"\bgoing to (?:be sick|throw up|faint)\b|\bi(?:'ve| have) (?:a |the )?(?:fever|flu|cold|headache|" + r"toothache|stomach ?ache)\b", re.I) +# moves that assert ONLY facts about Oracle7 itself may speak over a withheld decline (they make no claim about the world) +SELF_ONLY = {"role", "customer", "remark", "compliment", "self", "identity", "vocative"} +FUTURE = re.compile(r"\b(?:will|going to|gonna|next (?:week|month|year)|tomorrow|soon|plan(?:ning)? to|about to)\b", re.I) +ACK = {"happiness": "That's lovely to hear!", "sadness": "I'm sorry to hear that.", + "anger": "That sounds really frustrating.", "fear": "That sounds worrying.", + "surprise": "Really? That's unexpected.", "disgust": "Ugh, that sounds unpleasant."} +BACK = {"happiness": "What's the best part of it for you?", "sadness": "Is there anything I can help with?", + "anger": "What happened?", "fear": "What's worrying you most about it?", + "surprise": "How did that come about?", "disgust": "What happened?"} +STOPCAP = {"I", "I'm", "I'll", "I've", "I'd", "We", "My", "Our", "The", "A", "An", "It", "This", "That", "Yes", "No", "Hi", + "Hello", "Hey", "Well", "Oh", "So", "And", "But", "Excuse", "Sorry", "Please", "Guess", "You", "Mom", "Dad"} + + +def native_state(n_passages): + """the open end: a state named by its structure, with its measurement""" + if n_passages <= 0: + return "unbound - it binds to nothing I've read, so it's new to me" + if n_passages < 4: + return "thinly bound - %d passage%s I've read" % (n_passages, "" if n_passages == 1 else "s") + return "resonant - it binds to %d passages I've read" % n_passages + + +class SelfOrgan: + def __init__(self, conv): + self.conv = conv # the ConverseOrgan: page_for / ground / definition / topic + self.sessions = {} # autobiography: session -> {"turns", "subjects", "feeling"} + + # ---- the store, read for the self ------------------------------------------------------------------------------- + def bound(self, phrase): + """how many passages the store binds this subject to (the resonance count), 0 when it binds to nothing""" + try: + pid = self.conv.page_for(phrase) + return len([s for _, s in self.conv.r.page(pid, 40) if len(s.split()) >= 6]) if pid else 0 + except Exception: # noqa: BLE001 + return 0 + + def named_thing(self, t): + """the place or thing a disclosure names: a capitalised name first, else the turn's subject""" + for m in re.finditer(r"(? """ +from __future__ import annotations + +import hashlib +import json +import re +import sys + +import numpy as np + +D = 1024 +LEADS = (("how many", "howmany"), ("how much", "howmuch"), ("who", "who"), ("whom", "who"), ("whose", "who"), ("where", "where"), + ("when", "when"), ("which", "which"), ("what", "what"), ("in what", "what"), ("in which", "which"), ("name", "name"), + ("tell me", "name"), ("the ", "the")) + + +def osub_seed(text: str, dim: int = D) -> np.ndarray: + h = int(hashlib.sha256(text.lower().strip().encode()).hexdigest(), 16) + rng = np.random.RandomState(h % (2 ** 31)) + return np.exp(1j * rng.uniform(0, 2 * np.pi, dim)).astype(np.complex64) + + +def bind(a, b): + return np.fft.ifft(np.fft.fft(a) * np.fft.fft(b)).astype(np.complex64) + + +ROLES = {r: osub_seed("o7pm-role:" + r) for r in ("wh", "rel", "num", "route")} + + +def lead(q): + t = " ".join(str(q).lower().split()) + for a, b in LEADS: + if t.startswith(a): + return b + return "other" + + +_READERS = None + + +def relation_key(q): + """the relation reader that recognises the question (the readers' own compiled phrasings), else 'none'""" + global _READERS + try: + import relation_readers_v1 as RD + import relation_spec_v2 as S + except Exception: # noqa: BLE001 + return "none" + if _READERS is None: + _READERS = [(k, S.compiled(k)) for k in RD.PROPS] + s, low = S.normalise(q) + for k, comp in _READERS: + for d in ("fwd", "inv"): + if any(rx.match(low) for rx in comp[d]): + return str(k) + return "none" + + +def route_of(path): + p = str(path or "") + p = re.sub(r"^(?:chat:resolved>|chat:and_parts>[^>]*>|specialist_precedence>)+", "", p) + if p.startswith("specialist_call>"): + m = re.search(r"specialist:([a-z0-9-]+)", p) + sid = m.group(1) if m else "x" + return "specialist:rel" if sid.startswith("rel-") else "specialist:" + re.sub(r"-(v\d+|r\d+)$", "", sid) + if p.startswith("precision_guard:"): + return "guard" + return re.split(r"[>|]", p, 1)[0] or "none" + + +def shape(q): + return {"wh": lead(q), "rel": relation_key(q), "num": "1" if re.search(r"\d", q) else "0"} + + +def path_vec(sh, route): + v = sum(bind(ROLES[r], osub_seed("o7pm:%s:%s" % (r, sh[r]))) for r in ("wh", "rel", "num")) + v = v + bind(ROLES["route"], osub_seed("o7pm:route:" + route)) + return (v / np.maximum(np.abs(v), 1e-8)).astype(np.complex64) + + +def key(sh, route): + return "%s|%s|%s|%s" % (sh["wh"], sh["rel"], sh["num"], route) + + +class PathMemory: + def __init__(self, entries=None): + self.entries = entries or {} # key -> {"shape": {...}, "route": r, "right": n, "wrong": n} + self._vec = {} + + def credit(self, sh, route, outcome): + k = key(sh, route) + e = self.entries.setdefault(k, {"shape": dict(sh), "route": route, "right": 0, "wrong": 0}) + e["right" if outcome == "right" else "wrong"] += 1 + + def _v(self, k): + if k not in self._vec: + e = self.entries[k] + self._vec[k] = path_vec(e["shape"], e["route"]) + return self._vec[k] + + def recall(self, sh, route, floor=0.45): + """-> (right, wrong, resonance, key used) for this path; exact if stored, else the most resonant stored path with the + same route above the floor, else None""" + k = key(sh, route) + if k in self.entries: + e = self.entries[k] + return e["right"], e["wrong"], 1.0, k + q = path_vec(sh, route) + best, arg = -1.0, None + for kk, e in self.entries.items(): + if e["route"] != route: + continue + r = float(np.real(np.vdot(self._v(kk), q)) / D) + if r > best: + best, arg = r, kk + if arg is None or best < floor: + return None + e = self.entries[arg] + return e["right"], e["wrong"], best, arg + + def precision(self, sh, route, prior=(1, 1), min_n=6): + r = self.recall(sh, route) + if r is None or r[0] + r[1] < min_n: + return None + return (r[0] + prior[0]) / (r[0] + r[1] + prior[0] + prior[1]), r[0] + r[1] + + def save(self, path): + json.dump({"D": D, "entries": self.entries}, open(path, "w"), indent=0) + + @classmethod + def load(cls, path): + return cls(json.load(open(path))["entries"]) + + +def build(files, out): + pm = PathMemory() + n = 0 + for f in files: + for line in open(f): + r = json.loads(line) + sh = shape(r["q"]) + for arm in ("head", "family"): + x = r.get(arm) + if not x or x.get("status") != "answered" or x.get("grade") not in ("right", "wrong"): + continue + pm.credit(sh, route_of(x["path"]), x["grade"]) + n += 1 + pm.save(out) + return pm, n + + +if __name__ == "__main__": + sys.path.insert(0, __import__("os").environ.get("ORACLE7_ROOT", "/opt/oracle-clm") + "/wander-train") # the runtime's relation readers - the shape must match the runtime's + import relation_readers_v1 # noqa: F401,E402 - fail loudly: a memory built without relation keys is a different memory + if sys.argv[1] == "build": + pm, n = build(sys.argv[2:-1], sys.argv[-1]) + print("credited outcomes", n, "paths", len(pm.entries)) diff --git a/root/wander-train/pubchem_periodic_table.csv b/root/wander-train/pubchem_periodic_table.csv new file mode 100644 index 0000000000000000000000000000000000000000..4185845f8ebd7bb9747d3aef3c6421b9014ca74f --- /dev/null +++ b/root/wander-train/pubchem_periodic_table.csv @@ -0,0 +1,119 @@ +"AtomicNumber","Symbol","Name","AtomicMass","CPKHexColor","ElectronConfiguration","Electronegativity","AtomicRadius","IonizationEnergy","ElectronAffinity","OxidationStates","StandardState","MeltingPoint","BoilingPoint","Density","GroupBlock","YearDiscovered" +1,"H","Hydrogen",1.0080,"FFFFFF","1s1",2.2,120,13.598,0.754,"+1, -1","Gas",13.81,20.28,0.00008988,"Nonmetal",1766 +2,"He","Helium",4.00260,"D9FFFF","1s2","",140,24.587,"","0","Gas",0.95,4.22,0.0001785,"Noble gas",1868 +3,"Li","Lithium",7.0,"CC80FF","[He]2s1",0.98,182,5.392,0.618,"+1","Solid",453.65,1615,0.534,"Alkali metal",1817 +4,"Be","Beryllium",9.012183,"C2FF00","[He]2s2",1.57,153,9.323,"","+2","Solid",1560,2744,1.85,"Alkaline earth metal",1798 +5,"B","Boron",10.81,"FFB5B5","[He]2s2 2p1",2.04,192,8.298,0.277,"+3","Solid",2348,4273,2.37,"Metalloid",1808 +6,"C","Carbon",12.011,"909090","[He]2s2 2p2",2.55,170,11.260,1.263,"+4, +2, -4","Solid",3823,4098,2.2670,"Nonmetal","Ancient" +7,"N","Nitrogen",14.007,"3050F8","[He] 2s2 2p3",3.04,155,14.534,"","+5, +4, +3, +2, +1, -1, -2, -3","Gas",63.15,77.36,0.0012506,"Nonmetal",1772 +8,"O","Oxygen",15.999,"FF0D0D","[He]2s2 2p4",3.44,152,13.618,1.461,"-2","Gas",54.36,90.2,0.001429,"Nonmetal",1774 +9,"F","Fluorine",18.99840316,"90E050","[He]2s2 2p5",3.98,135,17.423,3.339,"-1","Gas",53.53,85.03,0.001696,"Halogen",1670 +10,"Ne","Neon",20.180,"B3E3F5","[He]2s2 2p6","",154,21.565,"","0","Gas",24.56,27.07,0.0008999,"Noble gas",1898 +11,"Na","Sodium",22.9897693,"AB5CF2","[Ne]3s1",0.93,227,5.139,0.548,"+1","Solid",370.95,1156,0.97,"Alkali metal",1807 +12,"Mg","Magnesium",24.305,"8AFF00","[Ne]3s2",1.31,173,7.646,"","+2","Solid",923,1363,1.74,"Alkaline earth metal",1808 +13,"Al","Aluminum",26.981538,"BFA6A6","[Ne]3s2 3p1",1.61,184,5.986,0.441,"+3","Solid",933.437,2792,2.70,"Post-transition metal","Ancient" +14,"Si","Silicon",28.085,"F0C8A0","[Ne]3s2 3p2",1.9,210,8.152,1.385,"+4, +2, -4","Solid",1687,3538,2.3296,"Metalloid",1854 +15,"P","Phosphorus",30.97376200,"FF8000","[Ne]3s2 3p3",2.19,180,10.487,0.746,"+5, +3, -3","Solid",317.3,553.65,1.82,"Nonmetal",1669 +16,"S","Sulfur",32.07,"FFFF30","[Ne]3s2 3p4",2.58,180,10.360,2.077,"+6, +4, -2","Solid",388.36,717.75,2.067,"Nonmetal","Ancient" +17,"Cl","Chlorine",35.45,"1FF01F","[Ne]3s2 3p5",3.16,175,12.968,3.617,"+7, +5, +1, -1","Gas",171.65,239.11,0.003214,"Halogen",1774 +18,"Ar","Argon",39.9,"80D1E3","[Ne]3s2 3p6","",188,15.760,"","0","Gas",83.8,87.3,0.0017837,"Noble gas",1894 +19,"K","Potassium",39.0983,"8F40D4","[Ar]4s1",0.82,275,4.341,0.501,"+1","Solid",336.53,1032,0.89,"Alkali metal",1807 +20,"Ca","Calcium",40.08,"3DFF00","[Ar]4s2",1,231,6.113,"","+2","Solid",1115,1757,1.54,"Alkaline earth metal","Ancient" +21,"Sc","Scandium",44.95591,"E6E6E6","[Ar]4s2 3d1",1.36,211,6.561,0.188,"+3","Solid",1814,3109,2.99,"Transition metal",1879 +22,"Ti","Titanium",47.867,"BFC2C7","[Ar]4s2 3d2",1.54,187,6.828,0.079,"+4, +3, +2","Solid",1941,3560,4.5,"Transition metal",1791 +23,"V","Vanadium",50.9415,"A6A6AB","[Ar]4s2 3d3",1.63,179,6.746,0.525,"+5, +4, +3, +2","Solid",2183,3680,6.0,"Transition metal",1801 +24,"Cr","Chromium",51.996,"8A99C7","[Ar]3d5 4s1",1.66,189,6.767,0.666,"+6, +3, +2","Solid",2180,2944,7.15,"Transition metal",1797 +25,"Mn","Manganese",54.93804,"9C7AC7","[Ar]4s2 3d5",1.55,197,7.434,"","+7, +4, +3, +2","Solid",1519,2334,7.3,"Transition metal",1774 +26,"Fe","Iron",55.84,"E06633","[Ar]4s2 3d6",1.83,194,7.902,0.163,"+3, +2","Solid",1811,3134,7.874,"Transition metal","Ancient" +27,"Co","Cobalt",58.93319,"F090A0","[Ar]4s2 3d7",1.88,192,7.881,0.661,"+3, +2","Solid",1768,3200,8.86,"Transition metal",1735 +28,"Ni","Nickel",58.693,"50D050","[Ar]4s2 3d8",1.91,163,7.640,1.156,"+3, +2","Solid",1728,3186,8.912,"Transition metal",1751 +29,"Cu","Copper",63.55,"C88033","[Ar]4s1 3d10",1.9,140,7.726,1.228,"+2, +1","Solid",1357.77,2835,8.933,"Transition metal","Ancient" +30,"Zn","Zinc",65.4,"7D80B0","[Ar]4s2 3d10",1.65,139,9.394,"","+2","Solid",692.68,1180,7.134,"Transition metal",1746 +31,"Ga","Gallium",69.723,"C28F8F","[Ar]4s2 3d10 4p1",1.81,187,5.999,0.3,"+3","Solid",302.91,2477,5.91,"Post-transition metal",1875 +32,"Ge","Germanium",72.63,"668F8F","[Ar]4s2 3d10 4p2",2.01,211,7.900,1.35,"+4, +2","Solid",1211.4,3106,5.323,"Metalloid",1886 +33,"As","Arsenic",74.92159,"BD80E3","[Ar]4s2 3d10 4p3",2.18,185,9.815,0.81,"+5, +3, -3","Solid",1090,887,5.776,"Metalloid","Ancient" +34,"Se","Selenium",78.97,"FFA100","[Ar]4s2 3d10 4p4",2.55,190,9.752,2.021,"+6, +4, -2","Solid",493.65,958,4.809,"Nonmetal",1817 +35,"Br","Bromine",79.90,"A62929","[Ar]4s2 3d10 4p5",2.96,183,11.814,3.365,"+5, +1, -1","Liquid",265.95,331.95,3.11,"Halogen",1826 +36,"Kr","Krypton",83.80,"5CB8D1","[Ar]4s2 3d10 4p6",3,202,14.000,"","0","Gas",115.79,119.93,0.003733,"Noble gas",1898 +37,"Rb","Rubidium",85.468,"702EB0","[Kr]5s1",0.82,303,4.177,0.468,"+1","Solid",312.46,961,1.53,"Alkali metal",1861 +38,"Sr","Strontium",87.62,"00FF00","[Kr]5s2",0.95,249,5.695,"","+2","Solid",1050,1655,2.64,"Alkaline earth metal",1790 +39,"Y","Yttrium",88.90584,"94FFFF","[Kr]5s2 4d1",1.22,219,6.217,0.307,"+3","Solid",1795,3618,4.47,"Transition metal",1794 +40,"Zr","Zirconium",91.22,"94E0E0","[Kr]5s2 4d2",1.33,186,6.634,0.426,"+4","Solid",2128,4682,6.52,"Transition metal",1789 +41,"Nb","Niobium",92.90637,"73C2C9","[Kr]5s1 4d4",1.6,207,6.759,0.893,"+5, +3","Solid",2750,5017,8.57,"Transition metal",1801 +42,"Mo","Molybdenum",95.95,"54B5B5","[Kr]5s1 4d5",2.16,209,7.092,0.746,"+6","Solid",2896,4912,10.2,"Transition metal",1778 +43,"Tc","Technetium",96.90636,"3B9E9E","[Kr]5s2 4d5",1.9,209,7.28,0.55,"+7, +6, +4","Solid",2430,4538,11,"Transition metal",1937 +44,"Ru","Ruthenium",101.1,"248F8F","[Kr]5s1 4d7",2.2,207,7.361,1.05,"+3","Solid",2607,4423,12.1,"Transition metal",1827 +45,"Rh","Rhodium",102.9055,"0A7D8C","[Kr]5s1 4d8",2.28,195,7.459,1.137,"+3","Solid",2237,3968,12.4,"Transition metal",1803 +46,"Pd","Palladium",106.42,"6985","[Kr]4d10",2.2,202,8.337,0.557,"+3, +2","Solid",1828.05,3236,12.0,"Transition metal",1803 +47,"Ag","Silver",107.868,"C0C0C0","[Kr]5s1 4d10",1.93,172,7.576,1.302,"+1","Solid",1234.93,2435,10.501,"Transition metal","Ancient" +48,"Cd","Cadmium",112.41,"FFD98F","[Kr]5s2 4d10",1.69,158,8.994,"","+2","Solid",594.22,1040,8.69,"Transition metal",1817 +49,"In","Indium",114.818,"A67573","[Kr]5s2 4d10 5p1",1.78,193,5.786,0.3,"+3","Solid",429.75,2345,7.31,"Post-transition metal",1863 +50,"Sn","Tin",118.71,"668080","[Kr]5s2 4d10 5p2",1.96,217,7.344,1.2,"+4, +2","Solid",505.08,2875,7.287,"Post-transition metal","Ancient" +51,"Sb","Antimony",121.760,"9E63B5","[Kr]5s2 4d10 5p3",2.05,206,8.64,1.07,"+5, +3, -3","Solid",903.78,1860,6.685,"Metalloid","Ancient" +52,"Te","Tellurium",127.6,"D47A00","[Kr]5s2 4d10 5p4",2.1,206,9.010,1.971,"+6, +4, -2","Solid",722.66,1261,6.232,"Metalloid",1782 +53,"I","Iodine",126.9045,"940094","[Kr]5s2 4d10 5p5",2.66,198,10.451,3.059,"+7, +5, +1, -1","Solid",386.85,457.55,4.93,"Halogen",1811 +54,"Xe","Xenon",131.29,"429EB0","[Kr]5s2 4d10 5p6",2.6,216,12.130,"","0","Gas",161.36,165.03,0.005887,"Noble gas",1898 +55,"Cs","Cesium",132.9054520,"57178F","[Xe]6s1",0.79,343,3.894,0.472,"+1","Solid",301.59,944,1.93,"Alkali metal",1860 +56,"Ba","Barium",137.33,"00C900","[Xe]6s2",0.89,268,5.212,"","+2","Solid",1000,2170,3.62,"Alkaline earth metal",1808 +57,"La","Lanthanum",138.9055,"70D4FF","[Xe]6s2 5d1",1.1,240,5.577,0.5,"+3","Solid",1191,3737,6.15,"Lanthanide",1839 +58,"Ce","Cerium",140.116,"FFFFC7","[Xe]6s2 4f1 5d1",1.12,235,5.539,0.5,"+4, +3","Solid",1071,3697,6.770,"Lanthanide",1803 +59,"Pr","Praseodymium",140.90766,"D9FFC7","[Xe]6s2 4f3",1.13,239,5.464,"","+3","Solid",1204,3793,6.77,"Lanthanide",1885 +60,"Nd","Neodymium",144.24,"C7FFC7","[Xe]6s2 4f4",1.14,229,5.525,"","+3","Solid",1294,3347,7.01,"Lanthanide",1885 +61,"Pm","Promethium",144.91276,"A3FFC7","[Xe]6s2 4f5","",236,5.55,"","+3","Solid",1315,3273,7.26,"Lanthanide",1945 +62,"Sm","Samarium",150.4,"8FFFC7","[Xe]6s2 4f6",1.17,229,5.644,"","+3, +2","Solid",1347,2067,7.52,"Lanthanide",1879 +63,"Eu","Europium",151.964,"61FFC7","[Xe]6s2 4f7","",233,5.670,"","+3, +2","Solid",1095,1802,5.24,"Lanthanide",1901 +64,"Gd","Gadolinium",157.25,"45FFC7","[Xe]6s2 4f7 5d1",1.2,237,6.150,"","+3","Solid",1586,3546,7.90,"Lanthanide",1880 +65,"Tb","Terbium",158.92535,"30FFC7","[Xe]6s2 4f9","",221,5.864,"","+3","Solid",1629,3503,8.23,"Lanthanide",1843 +66,"Dy","Dysprosium",162.500,"1FFFC7","[Xe]6s2 4f10",1.22,229,5.939,"","+3","Solid",1685,2840,8.55,"Lanthanide",1886 +67,"Ho","Holmium",164.93033,"00FF9C","[Xe]6s2 4f11",1.23,216,6.022,"","+3","Solid",1747,2973,8.80,"Lanthanide",1878 +68,"Er","Erbium",167.26,"","[Xe]6s2 4f12",1.24,235,6.108,"","+3","Solid",1802,3141,9.07,"Lanthanide",1843 +69,"Tm","Thulium",168.93422,"00D452","[Xe]6s2 4f13",1.25,227,6.184,"","+3","Solid",1818,2223,9.32,"Lanthanide",1879 +70,"Yb","Ytterbium",173.05,"00BF38","[Xe]6s2 4f14","",242,6.254,"","+3, +2","Solid",1092,1469,6.90,"Lanthanide",1878 +71,"Lu","Lutetium",174.9667,"00AB24","[Xe]6s2 4f14 5d1",1.27,221,5.426,"","+3","Solid",1936,3675,9.84,"Lanthanide",1907 +72,"Hf","Hafnium",178.49,"4DC2FF","[Xe]6s2 4f14 5d2",1.3,212,6.825,"","+4","Solid",2506,4876,13.3,"Transition metal",1923 +73,"Ta","Tantalum",180.9479,"4DA6FF","[Xe]6s2 4f14 5d3",1.5,217,7.89,0.322,"+5","Solid",3290,5731,16.4,"Transition metal",1802 +74,"W","Tungsten",183.84,"2194D6","[Xe]6s2 4f14 5d4",2.36,210,7.98,0.815,"+6","Solid",3695,5828,19.3,"Transition metal",1783 +75,"Re","Rhenium",186.207,"267DAB","[Xe]6s2 4f14 5d5",1.9,217,7.88,0.15,"+7, +6, +4","Solid",3459,5869,20.8,"Transition metal",1925 +76,"Os","Osmium",190.2,"266696","[Xe]6s2 4f14 5d6",2.2,216,8.7,1.1,"+4, +3","Solid",3306,5285,22.57,"Transition metal",1803 +77,"Ir","Iridium",192.22,"175487","[Xe]6s2 4f14 5d7",2.2,202,9.1,1.565,"+4, +3","Solid",2719,4701,22.42,"Transition metal",1803 +78,"Pt","Platinum",195.08,"D0D0E0","[Xe]6s1 4f14 5d9",2.28,209,9,2.128,"+4, +2","Solid",2041.55,4098,21.46,"Transition metal",1735 +79,"Au","Gold",196.96657,"FFD123","[Xe]6s1 4f14 5d10",2.54,166,9.226,2.309,"+3, +1","Solid",1337.33,3129,19.282,"Transition metal","Ancient" +80,"Hg","Mercury",200.59,"B8B8D0","[Xe]6s2 4f14 5d10",2,209,10.438,"","+2, +1","Liquid",234.32,629.88,13.5336,"Transition metal","Ancient" +81,"Tl","Thallium",204.383,"A6544D","[Xe]6s2 4f14 5d10 6p1",1.62,196,6.108,0.2,"+3, +1","Solid",577,1746,11.8,"Post-transition metal",1861 +82,"Pb","Lead",207,"575961","[Xe]6s2 4f14 5d10 6p2",2.33,202,7.417,0.36,"+4, +2","Solid",600.61,2022,11.342,"Post-transition metal","Ancient" +83,"Bi","Bismuth",208.98040,"9E4FB5","[Xe]6s2 4f14 5d10 6p3",2.02,207,7.289,0.946,"+5, +3","Solid",544.55,1837,9.807,"Post-transition metal",1753 +84,"Po","Polonium",208.98243,"AB5C00","[Xe]6s2 4f14 5d10 6p4",2,197,8.417,1.9,"+4, +2","Solid",527,1235,9.32,"Metalloid",1898 +85,"At","Astatine",209.98715,"754F45","[Xe]6s2 4f14 5d10 6p5",2.2,202,9.5,2.8,"7, 5, 3, 1, -1","Solid",575,"",7,"Halogen",1940 +86,"Rn","Radon",222.01758,"428296","[Xe]6s2 4f14 5d10 6p6","",220,10.745,"","0","Gas",202,211.45,0.00973,"Noble gas",1900 +87,"Fr","Francium",223.01973,"420066","[Rn]7s1",0.7,348,3.9,0.47,"+1","Solid",300,"","","Alkali metal",1939 +88,"Ra","Radium",226.02541,"007D00","[Rn]7s2",0.9,283,5.279,"","+2","Solid",973,1413,5,"Alkaline earth metal",1898 +89,"Ac","Actinium",227.02775,"70ABFA","[Rn]7s2 6d1",1.1,260,5.17,"","+3","Solid",1324,3471,10.07,"Actinide",1899 +90,"Th","Thorium",232.038,"00BAFF","[Rn]7s2 6d2",1.3,237,6.08,"","+4","Solid",2023,5061,11.72,"Actinide",1828 +91,"Pa","Protactinium",231.03588,"00A1FF","[Rn]7s2 5f2 6d1",1.5,243,5.89,"","+5, +4","Solid",1845,"",15.37,"Actinide",1913 +92,"U","Uranium",238.0289,"008FFF","[Rn]7s2 5f3 6d1",1.38,240,6.194,"","+6, +5, +4, +3","Solid",1408,4404,18.95,"Actinide",1789 +93,"Np","Neptunium",237.048172,"0080FF","[Rn]7s2 5f4 6d1",1.36,221,6.266,"","+6, +5, +4, +3","Solid",917,4175,20.25,"Actinide",1940 +94,"Pu","Plutonium",244.06420,"006BFF","[Rn]7s2 5f6",1.28,243,6.06,"","+6, +5, +4, +3","Solid",913,3501,19.84,"Actinide",1940 +95,"Am","Americium",243.061380,"545CF2","[Rn]7s2 5f7",1.3,244,5.993,"","+6, +5, +4, +3","Solid",1449,2284,13.69,"Actinide",1944 +96,"Cm","Curium",247.07035,"785CE3","[Rn]7s2 5f7 6d1",1.3,245,6.02,"","+3","Solid",1618,3400,13.51,"Actinide",1944 +97,"Bk","Berkelium",247.07031,"8A4FE3","[Rn]7s2 5f9",1.3,244,6.23,"","+4, +3","Solid",1323,"",14,"Actinide",1949 +98,"Cf","Californium",251.07959,"A136D4","[Rn]7s2 5f10",1.3,245,6.30,"","+3","Solid",1173,"","","Actinide",1950 +99,"Es","Einsteinium",252.0830,"B31FD4","[Rn]7s2 5f11",1.3,245,6.42,"","+3","Solid",1133,"","","Actinide",1952 +100,"Fm","Fermium",257.09511,"B31FBA","[Rn] 5f12 7s2",1.3,"",6.50,"","+3","Solid",1800,"","","Actinide",1952 +101,"Md","Mendelevium",258.09843,"B30DA6","[Rn]7s2 5f13",1.3,"",6.58,"","+3, +2","Solid",1100,"","","Actinide",1955 +102,"No","Nobelium",259.10100,"BD0D87","[Rn]7s2 5f14",1.3,"",6.65,"","+3, +2","Solid",1100,"","","Actinide",1957 +103,"Lr","Lawrencium",266.120,"C70066","[Rn]7s2 5f14 6d1",1.3,"","","","+3","Solid",1900,"","","Actinide",1961 +104,"Rf","Rutherfordium",267.122,"CC0059","[Rn]7s2 5f14 6d2","","","","","+4","Solid","","","","Transition metal",1964 +105,"Db","Dubnium",268.126,"D1004F","[Rn]7s2 5f14 6d3","","","","","5, 4, 3","Solid","","","","Transition metal",1967 +106,"Sg","Seaborgium",269.128,"D90045","[Rn]7s2 5f14 6d4","","","","","6, 5, 4, 3, 0","Solid","","","","Transition metal",1974 +107,"Bh","Bohrium",270.133,"E00038","[Rn]7s2 5f14 6d5","","","","","7, 5, 4, 3","Solid","","","","Transition metal",1976 +108,"Hs","Hassium",269.1336,"E6002E","[Rn]7s2 5f14 6d6","","","","","8, 6, 5, 4, 3, 2","Solid","","","","Transition metal",1984 +109,"Mt","Meitnerium",277.154,"EB0026","[Rn]7s2 5f14 6d7 (calculated)","","","","","9, 8, 6, 4, 3, 1","Solid","","","","Transition metal",1982 +110,"Ds","Darmstadtium",282.166,"","[Rn]7s2 5f14 6d8 (predicted)","","","","","8, 6, 4, 2, 0","Expected to be a Solid","","","","Transition metal",1994 +111,"Rg","Roentgenium",282.169,"","[Rn]7s2 5f14 6d9 (predicted)","","","","","5, 3, 1, -1","Expected to be a Solid","","","","Transition metal",1994 +112,"Cn","Copernicium",286.179,"","[Rn]7s2 5f14 6d10 (predicted)","","","","","2, 1, 0","Expected to be a Solid","","","","Transition metal",1996 +113,"Nh","Nihonium",286.182,"","[Rn]5f14 6d10 7s2 7p1 (predicted)","","","","","","Expected to be a Solid","","","","Post-transition metal",2004 +114,"Fl","Flerovium",290.192,"","[Rn]7s2 7p2 5f14 6d10 (predicted)","","","","","6, 4,2, 1, 0","Expected to be a Solid","","","","Post-transition metal",1998 +115,"Mc","Moscovium",290.196,"","[Rn]7s2 7p3 5f14 6d10 (predicted)","","","","","3, 1","Expected to be a Solid","","","","Post-transition metal",2003 +116,"Lv","Livermorium",293.205,"","[Rn]7s2 7p4 5f14 6d10 (predicted)","","","","","+4, +2, -2","Expected to be a Solid","","","","Post-transition metal",2000 +117,"Ts","Tennessine",294.211,"","[Rn]7s2 7p5 5f14 6d10 (predicted)","","","","","+5, +3, +1, -1","Expected to be a Solid","","","","Halogen",2010 +118,"Og","Oganesson",295.216,"","[Rn]7s2 7p6 5f14 6d10 (predicted)","","","","","+6, +4, +2, +1, 0, -1","Expected to be a Gas","","","","Noble gas",2006 diff --git a/root/wander-train/relation_forms_v2.py b/root/wander-train/relation_forms_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..2fc694721ec7bfa58d882577ebe93ed2497abd81 --- /dev/null +++ b/root/wander-train/relation_forms_v2.py @@ -0,0 +1,118 @@ +"""relation_forms_v2 = v1 + forms for more store properties (see make_relation_forms_v2.py). +relation_forms_v1 - question FORMS that name their relation, answered through the typed relation store. + +"Where is the Colosseum?", "Who discovered penicillin?", "When was the Eiffel Tower built?" carry no relation words the +mined lexicon can match, so the three-tree runtime found no relation and declined. The form itself names the relation, +in Wikidata's own vocabulary. Each form lists its properties in order of SPECIFICITY and they are tried ONE AT A TIME +(oracle_tritree_v2 only=True), because located-in (Rome) and country (Italy) are both true of the Colosseum and a joint +search would see two answers and abstain. Every check of the runtime still applies: answer type, one value, the +corroborating sentence from the subject's own article (rule A), the verb licence for dates (rule B). +answer(tt, q) -> (runtime answer, form name) | (None, None)""" +from __future__ import annotations + +import re + +FORMS = [ + # v2 (2026-10-03): forms for more of the store's properties, most specific first + ("screenwriter", re.compile(r"\bwho\s+wrote\s+the\s+(?:screenplay|script)\b", re.I), [58]), + ("country", re.compile(r"\b(?:in\s+)?(?:which|what)\s+country\s+(?:is|are|was|were)\b(?!.*\b(?:born|from)\b)|" + r"\b(?:is|are)\s+(?:in|located\s+in)\s+(?:which|what)\s+country\b", re.I), [17]), + ("continent", re.compile(r"\b(?:on|in)\s+(?:which|what)\s+continent\b|\b(?:which|what)\s+continent\s+(?:is|are)\b", re.I), [30]), + ("creator", re.compile(r"\bwho\s+(?:painted|sculpted|drew|carved)\b", re.I), [170]), + ("performer", re.compile(r"\bwho\s+(?:sang|sings|performed|recorded)\b|\b(?:which|what)\s+(?:band|singer|group|rapper)\s+" + r"(?:released|recorded|sang|performed)\b", re.I), [175]), + ("producer", re.compile(r"\bwho\s+produced\b", re.I), [162]), + ("author_of", re.compile(r"\bwho\s+(?:is|was)\s+the\s+author\s+of\b", re.I), [50]), + ("spouse", re.compile(r"\bwho\s+(?:is|was)\s+.+?\s+married\s+to\b|\bwho\s+did\s+.+?\s+marry\b|" + r"\b(?:wife|husband|spouse)\s+of\b", re.I), [26]), + ("father", re.compile(r"\bwho\s+(?:is|was)\s+(?:the\s+)?father\s+of\b", re.I), [22]), + ("mother", re.compile(r"\bwho\s+(?:is|was)\s+(?:the\s+)?mother\s+of\b", re.I), [25]), + ("deathplace", re.compile(r"\bwhere\s+did\s+.+?\s+die\b", re.I), [20]), + ("nationality", re.compile(r"\bnationality\s+of\b|\bwhat\s+(?:is|was)\s+.+?(?:'s)?\s+nationality\b", re.I), [27]), + ("instrument", re.compile(r"\bwhat\s+instrument\s+(?:does|did)\b", re.I), [1303]), + ("sport", re.compile(r"\bwhat\s+sport\s+(?:does|did)\b", re.I), [641]), + ("headquarters", re.compile(r"\bheadquarter(?:s|ed)\b", re.I), [159]), + ("named_after", re.compile(r"\bnamed\s+after\s+(?:who|whom|what)\b|\bwho\s+(?:is|was)\s+.+?\s+named\s+after\b", re.I), [138]), + ("published", re.compile(r"\bwhen\s+was\s+.+?\s+(?:published|released)\b", re.I), [577]), + ("where", re.compile(r"^\s*where\s+(?:is|are|was|were|does|do)\b", re.I), [131, 276, 17, 30]), + ("discoverer", re.compile(r"\bwho\s+(?:first\s+)?(?:discovered|invented)\b", re.I), [61]), + ("architect", re.compile(r"\bwho\s+designed\b", re.I), [84, 170]), # "built" is not "designed" + ("founder", re.compile(r"\bwho\s+founded\b", re.I), [112]), + ("author", re.compile(r"\bwho\s+(?:wrote|authored)\b", re.I), [50]), + ("director", re.compile(r"\bwho\s+directed\b", re.I), [57]), + ("composer", re.compile(r"\bwho\s+composed\b", re.I), [86]), + ("inception", re.compile(r"\bwhen\s+(?:was|were|did)\b.*\b(?:built|founded|established|completed|opened|created|formed)\b", + re.I), [571]), + ("capital", re.compile(r"\bcapital\s+(?:city\s+)?of\b", re.I), [36]), + ("language", re.compile(r"\bwhat\s+(?:language|languages)\s+(?:is|are|do)\b.*\bspoken\b|\bofficial\s+language\s+of\b", + re.I), [37]), + ("currency", re.compile(r"\bcurrency\s+(?:of|in|does)\b", re.I), [38]), + ("birth", re.compile(r"\bwhen\s+was\s+.+?\s+born\b", re.I), [569]), + ("birthplace", re.compile(r"\bwhere\s+was\s+.+?\s+born\b", re.I), [19]), + ("death", re.compile(r"\bwhen\s+did\s+.+?\s+die\b", re.I), [570]), +] + + +def form_of(q): + # a more specific form wins over the generic "where" (where was X born -> birthplace) + hits = [(name, props) for name, rx, props in FORMS if rx.search(q or "")] + if not hits: + return None + specific = [h for h in hits if h[0] != "where"] + return (specific or hits)[0] + + +ARG = { # where each form's ARGUMENT (the entity asked about) sits + "where": r"^\s*where\s+(?:is|are|was|were|does|do)\s+(?:the\s+)?(?P.+?)(?:\s+(?:located|from|situated|found|live|lives|based))?\s*\??$", + "discoverer": r"\b(?:discovered|invented)\s+(?:the\s+)?(?P.+?)\s*\??$", + "architect": r"\bdesigned\s+(?:the\s+)?(?P.+?)\s*\??$", + "founder": r"\bfounded\s+(?:the\s+)?(?P.+?)\s*\??$", + "author": r"\b(?:wrote|authored)\s+(?:the\s+)?(?P.+?)\s*\??$", + "director": r"\bdirected\s+(?:the\s+)?(?P.+?)\s*\??$", + "composer": r"\bcomposed\s+(?:the\s+)?(?P.+?)\s*\??$", + "inception": r"\bwhen\s+(?:was|were|did)\s+(?:the\s+)?(?P.+?)\s+(?:built|founded|established|completed|opened|created|formed)\b", + "capital": r"\bcapital\s+(?:city\s+)?of\s+(?:the\s+)?(?P.+?)\s*\??$", + "language": r"\b(?:spoken\s+in|language\s+of|in)\s+(?:the\s+)?(?P[^?]+?)\s*\??$", + "currency": r"\bcurrency\s+(?:of|in|does)\s+(?:the\s+)?(?P.+?)(?:\s+use)?\s*\??$", + "birth": r"\bwhen\s+was\s+(?P.+?)\s+born\b", + "birthplace": r"\bwhere\s+was\s+(?P.+?)\s+born\b", + "death": r"\bwhen\s+did\s+(?P.+?)\s+die\b", +} + + +def truecase(tt, q, name): + """a lowercase argument ("jamarcus russell") is replaced by the store's own title for it ("JaMarcus Russell"): the + reader links only capitalised spans, and people type lowercase. Only the form's ARGUMENT slot is touched.""" + rx = ARG.get(name) + m = re.search(rx, q, re.I) if rx else None + if not m: + return q + x = m.group("x").strip(" ?.") + if not x or x[:1].isupper(): + return q + for cand in (x, re.sub(r"^(?:the|a|an)\s+", "", x, flags=re.I)): + try: + qid = tt.rs.entity(cand) + except Exception: # noqa: BLE001 + qid = None + if qid: + title = tt.rs.title(qid) or tt.rs.label(qid) + if title and title.lower().replace("_", " ") == cand.lower(): + return q[:m.start("x")] + title + q[m.end("x"):] + return q + + +def answer(tt, q): + f = form_of(q) + if not f: + return None, None + name, props = f + q = truecase(tt, q, name) + for p in props: + try: + a = tt.answer(q, form_props={("fwd", p): 1}, only=True) + except Exception: # noqa: BLE001 + a = None + if a: + return a, name + return None, None diff --git a/root/wander-train/stage5c58_cycling_cognitive_substrate.py b/root/wander-train/stage5c58_cycling_cognitive_substrate.py new file mode 100644 index 0000000000000000000000000000000000000000..a4aa14b50157723d56ccb423bd40fa80f9bad6c6 --- /dev/null +++ b/root/wander-train/stage5c58_cycling_cognitive_substrate.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +"""C58 typed bridge from C54-C57 conversation into the Stage 5M substrate. + +The mutable store never enters a reasoning rollout. A turn receives an +immutable, handle-only snapshot; opaque handles traverse a stabilized HRR +selector; lexical payloads resolve only at Malkuth. Recalled hypothetical +claims remain user premises. Only explicitly provenance-locked semantic +records are eligible to be treated as authenticated facts. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import Any, Mapping, Sequence + +import numpy as np + +from ensouled._substrate.oracle_memory import ( + OraclePersistentSubstrate, + SnapshotBundle, + canonical_json, + digest_value, +) +from oracle_10d_resolution_state import array_sha256 +from stage5m_persistent_substrate_bridge import ( + attach_memory_ingress, + release_memory_at_malkuth, +) +from train_stage2a_role_ladder import HRR_DIM, cosine, normalize, unitary + + +SCHEMA = "oracle-stage5c58-cycling-cognitive-substrate-v1" +MEMORY_CLASSES = ( + "hypothetical_reasoning_episode", + "verified_factual_response_episode", + "provisional_definition", + "conversation_correction", + "unresolved_alternative", + "authenticated_fact", +) +PROVISIONAL_CLASSES = frozenset({ + "hypothetical_reasoning_episode", + "provisional_definition", + "conversation_correction", + "unresolved_alternative", +}) + + +def _surface_map(dialogue: Any) -> dict[str, str]: + resolver = dialogue.c56.resolver + session = dialogue.c56.session + values = {symbol: surface for surface, symbol in resolver.base_symbols.items()} + values.update({symbol: surface for surface, symbol in session.local_symbols.items()}) + return values + + +def _event_key(prefix: str, payload: Mapping[str, Any]) -> str: + return f"c58:{prefix}:{digest_value(payload)}" + + +@dataclass(frozen=True) +class MemoryTransitAudit: + accepted: bool + selected_handle_id: str | None + expected_handle_id: str | None + minimum_cleanup_margin: float + moving_state_sha256: str + spine_state_sha256: str + use_spine: bool + + def as_dict(self) -> dict[str, Any]: + return { + "accepted": self.accepted, + "selected_handle_id": self.selected_handle_id, + "expected_handle_id": self.expected_handle_id, + "minimum_cleanup_margin": self.minimum_cleanup_margin, + "moving_state_sha256": self.moving_state_sha256, + "spine_state_sha256": self.spine_state_sha256, + "stabilizing_spine_used": self.use_spine, + "plaintext_payloads_exposed": 0, + "pre_malkuth_unbinds": 0, + } + + +class OpaqueHandleTransit: + """Select one immutable snapshot handle without inspecting its payload.""" + + def __init__(self, *, stabilization_gain: float = 4.0): + self.stabilization_gain = float(stabilization_gain) + + @staticmethod + def _state(handle_id: str) -> np.ndarray: + return unitary(f"stage5c58:opaque-memory-handle:{handle_id}") + + def select( + self, + bundle: SnapshotBundle, + *, + use_spine: bool = True, + expected_index: int = 0, + ) -> MemoryTransitAudit: + handles = bundle.snapshot.handles + if not handles: + empty = unitary("stage5c58:empty-memory-transit") + return MemoryTransitAudit( + False, None, None, 0.0, array_sha256(empty), array_sha256(empty), use_spine, + ) + states = tuple(self._state(value.handle_id) for value in handles) + weighted = np.zeros(HRR_DIM, dtype=np.complex64) + for state, handle in zip(states, handles, strict=True): + weighted += np.complex64(0.25 + max(0.0, handle.score)) * state + moving = normalize(weighted) + expected = handles[expected_index].handle_id + spine = states[expected_index] + probe = moving + if use_spine: + probe = normalize(probe + np.complex64(self.stabilization_gain) * spine) + scores = sorted( + ((cosine(probe, state), handle.handle_id) for state, handle in zip(states, handles, strict=True)), + reverse=True, + ) + selected = scores[0][1] + margin = scores[0][0] - (scores[1][0] if len(scores) > 1 else 0.0) + return MemoryTransitAudit( + accepted=bool(selected == expected and margin > 0.50), + selected_handle_id=selected, + expected_handle_id=expected, + minimum_cleanup_margin=float(margin), + moving_state_sha256=array_sha256(moving), + spine_state_sha256=array_sha256(spine), + use_spine=use_spine, + ) + + +@dataclass(frozen=True) +class CognitiveRecall: + authorized: bool + reason: str + handle_id: str | None + bank: str | None + handle_status: str | None + memory_class: str | None + roles: tuple[tuple[str, str], ...] + provenance: tuple[tuple[str, Any], ...] + snapshot_digest: str | None + transit: MemoryTransitAudit + ingress_sha256: str | None + + def role_map(self) -> dict[str, str]: + return dict(self.roles) + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA, + "authorized": self.authorized, + "reason": self.reason, + "handle_id": self.handle_id, + "bank": self.bank, + "handle_status": self.handle_status, + "memory_class": self.memory_class, + "snapshot_digest": self.snapshot_digest, + "transit": self.transit.as_dict(), + "ingress_sha256": self.ingress_sha256, + "lexical_payload_released_at_malkuth": bool(self.authorized), + "pre_malkuth_payload_fields": 0, + } + + +class CyclingCognitiveSubstrate: + def __init__( + self, + store: OraclePersistentSubstrate, + *, + transit: OpaqueHandleTransit | None = None, + ): + self.store = store + self.transit = transit or OpaqueHandleTransit() + + def record_response(self, question: str, response: Mapping[str, Any], dialogue: Any) -> str: + surfaces = _surface_map(dialogue) + if response.get("action") == "hypothetical" and response.get( + "c57_inverse_verification", {} + ).get("accepted"): + germ = response["response_germ"] + claims = [] + for claim in germ["claims"]: + claims.append({ + "subject": surfaces[claim["subject_symbol"]], + "predicate": claim["predicate"], + "object": surfaces[claim["object_symbol"]], + "polarity": claim["polarity"], + "modality": claim["modality"], + "source_clause_sha256": claim["source_clause_sha256"], + }) + roles = { + "root": question, + "source": surfaces[germ["source_symbol"]], + "field": claims[0]["object"], + "transition": canonical_json([value["predicate"] for value in claims]), + "result": canonical_json([ + surfaces[value] for value in germ["consequence_symbols"] + ]), + "goal": germ["query_operation"], + "warrant": canonical_json(claims), + "action": "hypothetical", + "situation": "hypothetical_reasoning_episode", + "status": "verified_transform_nonfactual", + } + payload = { + "response_germ": germ["protected_invariant_sha256"], + "roles": roles, + } + return self.store.bind_event( + bank="episodic", + roles=roles, + source_record_id=f"c57:{germ['protected_invariant_sha256']}", + provenance={ + "schema": "oracle-stage5c57-verified-morphogenetic-reply-v1", + "response_germ_sha256": germ["protected_invariant_sha256"], + "graph_invariant_sha256": germ["graph_invariant_sha256"], + "inverse_verification_sha256": response["reply"]["verification"][ + "verification_sha256" + ], + "epistemic_class": "nonfactual_user_premise", + }, + status="verified", + confidence=1.0, + event_key=_event_key("hypothetical", payload), + metadata={"memory_class": "hypothetical_reasoning_episode"}, + ) + + if response.get("action") == "answer" and response.get( + "reply", {} + ).get("verification", {}).get("accepted"): + program = response.get("program", {}) + roles = { + "root": question, + "source": str(program.get("source_symbol", "")), + "goal": str(program.get("goal_symbol", "")), + "transition": canonical_json(program.get("relation_program", [])), + "result": str(response.get("endpoint", "")), + "warrant": canonical_json(program.get("fact_ids", [])), + "action": "answer", + "situation": "verified_factual_response_episode", + "status": "verified_response_not_fact_storage", + } + payload = {"program": program, "roles": roles} + return self.store.bind_event( + bank="episodic", + roles=roles, + source_record_id=f"c51:{response['reply'].get('authorization_sha256', '')}", + provenance={ + "schema": "oracle-stage5c51-verified-variable-science-reply-v1", + "fact_ids": program.get("fact_ids", []), + "authorization_sha256": response["reply"].get("authorization_sha256"), + "epistemic_class": "conversation_episode_not_fact_storage", + }, + status="verified", + confidence=1.0, + event_key=_event_key("factual_response", payload), + metadata={"memory_class": "verified_factual_response_episode"}, + ) + + roles = { + "root": question, + "result": str(response.get("reason", response.get("action", "unresolved"))), + "action": str(response.get("action", "clarify")), + "situation": "unresolved_alternative", + "status": "unverified", + } + payload = {"roles": roles, "alternatives": response.get("resolved_utterance", {}).get("alternatives", [])} + return self.store.bind_event( + bank="quarantine", + roles=roles, + source_record_id=f"c58:unresolved:{hashlib.sha256(question.encode('utf-8')).hexdigest()}", + provenance={"schema": str(response.get("schema_version", "unknown"))}, + status="unverified", + confidence=0.25, + event_key=_event_key("unresolved", payload), + metadata={"memory_class": "unresolved_alternative"}, + ) + + def record_provisional_definition( + self, + *, + term: str, + definition: str, + source_record_id: str, + ) -> str: + roles = { + "source": term, + "result": definition, + "situation": "provisional_definition", + "status": "unverified_user_definition", + } + return self.store.bind_event( + bank="episodic", + roles=roles, + source_record_id=source_record_id, + provenance={"epistemic_class": "provisional_user_definition"}, + status="unverified", + confidence=0.5, + event_key=_event_key("definition", roles), + metadata={"memory_class": "provisional_definition"}, + ) + + def record_correction( + self, + *, + topic: str, + correction: str, + supersedes_memory_id: str, + source_record_id: str, + ) -> str: + roles = { + "source": topic, + "correction": correction, + "situation": "conversation_correction", + "status": "unverified_user_correction", + } + return self.store.bind_event( + bank="episodic", + roles=roles, + source_record_id=source_record_id, + provenance={ + "epistemic_class": "provisional_user_correction", + "supersedes_memory_id": supersedes_memory_id, + }, + status="unverified", + confidence=0.6, + event_key=_event_key("correction", roles), + contradictions=(supersedes_memory_id,), + metadata={"memory_class": "conversation_correction"}, + ) + + def record_authenticated_fact( + self, + *, + source: str, + relation: str, + destination: str, + fact_id: str, + evidence_sha256: str, + source_document_sha256: str, + ) -> str: + if not fact_id or len(evidence_sha256) != 64 or len(source_document_sha256) != 64: + raise ValueError("authenticated C58 facts require immutable fact and evidence provenance") + roles = { + "source": source, + "transition": relation, + "result": destination, + "warrant": evidence_sha256, + "provenance": source_document_sha256, + "situation": "authenticated_fact", + "status": "verified", + } + return self.store.bind_event( + bank="semantic", + roles=roles, + source_record_id=fact_id, + provenance={ + "fact_id": fact_id, + "evidence_sha256": evidence_sha256, + "source_document_sha256": source_document_sha256, + "epistemic_class": "authenticated_fact", + }, + status="verified", + confidence=1.0, + event_key=_event_key("authenticated_fact", roles), + metadata={"memory_class": "authenticated_fact"}, + ) + + def recall( + self, + query_roles: Mapping[str, str], + *, + root_sha256: str, + banks: Sequence[str] = ("episodic",), + allowed_classes: Sequence[str] = ("hypothetical_reasoning_episode",), + top_k: int = 8, + request_nonce: str | None = None, + use_spine: bool = True, + ) -> CognitiveRecall: + bundle = self.store.create_snapshot( + query_roles, + top_k=top_k, + banks=banks, + request_nonce=request_nonce, + minimum_score=0.0, + ) + transit = self.transit.select(bundle, use_spine=use_spine) + if not transit.accepted or transit.selected_handle_id is None: + return CognitiveRecall( + False, "opaque_handle_transit_failed", None, None, None, None, + (), (), bundle.snapshot.snapshot_digest, transit, None, + ) + tree = { + "id": f"c58-memory-tree:{root_sha256[:20]}", + "protected_roles": {"root": root_sha256}, + "dual_rail_contract": { + "content_rail": {}, + "stabilizing_spine": { + "stage4:spine:root": root_sha256, + "stage4:spine:unbind_permission": "malkuth_only", + }, + }, + "payload_contract": { + "raw_payload_text_present": False, + "malkuth_only_resolution": True, + }, + } + augmented = attach_memory_ingress( + tree, bundle, expected_agent_id=self.store.agent_id, + ) + release = release_memory_at_malkuth( + augmented, + bundle, + transit.selected_handle_id, + malkuth_entered=True, + ) + handle = next( + value for value in bundle.snapshot.handles + if value.handle_id == transit.selected_handle_id + ) + roles = release.role_map() if release.authorized else {} + memory_class = roles.get("situation") + authorized = bool(release.authorized and memory_class in set(allowed_classes)) + reason = ( + "authorized" if authorized + else "memory_class_not_authorized" if release.authorized + else release.reason + ) + return CognitiveRecall( + authorized=authorized, + reason=reason, + handle_id=handle.handle_id, + bank=handle.bank, + handle_status=handle.status, + memory_class=memory_class, + roles=release.roles if authorized else (), + provenance=release.provenance if authorized else (), + snapshot_digest=bundle.snapshot.snapshot_digest, + transit=transit, + ingress_sha256=digest_value(augmented), + ) + + @staticmethod + def fact_release_eligible(recall: CognitiveRecall) -> bool: + if not recall.authorized: + return False + provenance = dict(recall.provenance) + return bool( + recall.bank == "semantic" + and recall.handle_status == "verified" + and recall.memory_class == "authenticated_fact" + and provenance.get("epistemic_class") == "authenticated_fact" + and provenance.get("fact_id") + and len(str(provenance.get("evidence_sha256", ""))) == 64 + and len(str(provenance.get("source_document_sha256", ""))) == 64 + ) + + def audit(self) -> dict[str, Any]: + inspected = self.store.inspect() + return { + "schema_version": SCHEMA, + "agent_id": self.store.agent_id, + "dim": inspected["dim"], + "state_version": inspected["version"], + "cycle_count": inspected["cycle_count"], + "state_digest": inspected["state_digest"], + "entries": inspected["entries"], + "implicit_norms": inspected["implicit_norms"], + "memory_classes": list(MEMORY_CLASSES), + "provisional_classes": sorted(PROVISIONAL_CLASSES), + "automatic_fact_promotion_paths": 0, + "mutable_store_enters_tree": False, + "malkuth_only_payload_release": True, + } diff --git a/root/wander-train/stage5h1_generic_dataset_expert.py b/root/wander-train/stage5h1_generic_dataset_expert.py new file mode 100644 index 0000000000000000000000000000000000000000..c3089d0b0dd4359da80ecab54c62655b3685d75d --- /dev/null +++ b/root/wander-train/stage5h1_generic_dataset_expert.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""Format-agnostic, provenance-locked bounded dataset expert compiler. + +The compiler does not infer semantics from unnamed columns. A small typed +schema identifies keys, field types, units, and optional relational views; +CSV, TSV, and JSONL rows then compile into sharded HRR field memories and +ordinary C43 expert facts without updating the carrier. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +import time +from collections import deque +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Mapping + +import numpy as np + +from stage5c43_corpus_expert import CorpusFact +from train_stage2a_role_ladder import HRR_DIM, bind, cosine, normalize, unbind, unitary + + +SCHEMA = "oracle-stage5h1-generic-dataset-expert-v1" +SPEC_SCHEMA = "oracle-stage5h1-generic-dataset-spec-v1" + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def canonical_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def canonical_scalar(value: Any, kind: str) -> str: + if kind == "string": + result = " ".join(str(value).strip().split()) + if not result: + raise ValueError("empty string field") + return result + if kind == "integer": + return str(int(value)) + if kind == "number": + number = float(value) + if not math.isfinite(number): + raise ValueError("non-finite numeric field") + return format(number, ".12g") + if kind == "boolean": + if isinstance(value, bool): + return "true" if value else "false" + lowered = str(value).strip().casefold() + if lowered not in {"true", "false", "1", "0", "yes", "no"}: + raise ValueError(f"invalid boolean: {value!r}") + return "true" if lowered in {"true", "1", "yes"} else "false" + raise ValueError(f"unsupported field type: {kind}") + + +def value_symbol(field: str, value: str) -> np.ndarray: + return unitary(f"stage5h1:value:{field}:{value}") + + +def key_symbol(expert_id: str, key: str) -> np.ndarray: + return normalize(bind( + unitary(f"stage5h1:expert:{expert_id}"), unitary(f"stage5h1:key:{key}"), + )) + + +@dataclass(frozen=True) +class FieldSpec: + name: str + kind: str + unit: str | None = None + + +@dataclass(frozen=True) +class EdgeView: + name: str + source_field: str + destination_field: str + relation_field: str | None = None + relation_constant: str | None = None + + +@dataclass(frozen=True) +class DatasetSchema: + expert_id: str + dataset_id: str + data_path: str + data_sha256: str + format: str + primary_key: tuple[str, ...] + fields: tuple[FieldSpec, ...] + competencies: tuple[str, ...] + source_url: str + source_split: str + edge_views: tuple[EdgeView, ...] + shard_rows: int = 48 + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "DatasetSchema": + edges = tuple(EdgeView( + name=str(item["name"]), source_field=str(item["source_field"]), + destination_field=str(item["destination_field"]), + relation_field=str(item["relation_field"]) if item.get("relation_field") else None, + relation_constant=str(item["relation_constant"]) if item.get("relation_constant") else None, + ) for item in value.get("edge_views", ())) + for edge in edges: + if bool(edge.relation_field) == bool(edge.relation_constant): + raise ValueError("edge view requires exactly one relation source") + return cls( + expert_id=str(value["expert_id"]), dataset_id=str(value["dataset_id"]), + data_path=str(value["data_path"]), data_sha256=str(value["data_sha256"]), + format=str(value["format"]), primary_key=tuple(str(item) for item in value["primary_key"]), + fields=tuple(FieldSpec(str(item["name"]), str(item["type"]), item.get("unit")) for item in value["fields"]), + competencies=tuple(str(item) for item in value.get("competencies", ())), + source_url=str(value["source_url"]), source_split=str(value.get("source_split", "train")), + edge_views=edges, shard_rows=int(value.get("shard_rows", 48)), + ) + + +def read_records(path: Path, format_name: str) -> list[dict[str, Any]]: + if format_name in {"csv", "tsv"}: + with path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle, delimiter="," if format_name == "csv" else "\t") + return [dict(row) for row in reader] + if format_name == "jsonl": + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + raise ValueError(f"unsupported bounded dataset format: {format_name}") + + +def normalize_records(schema: DatasetSchema, records: Iterable[Mapping[str, Any]]) -> tuple[dict[str, str], ...]: + field_map = {field.name: field for field in schema.fields} + if not schema.primary_key or any(value not in field_map for value in schema.primary_key): + raise ValueError("invalid primary key declaration") + for edge in schema.edge_views: + needed = {edge.source_field, edge.destination_field} | ({edge.relation_field} if edge.relation_field else set()) + if not needed <= set(field_map): + raise ValueError(f"edge view {edge.name} references an unknown field") + normalized = [] + seen = {} + for ordinal, record in enumerate(records): + if set(record) != set(field_map): + raise ValueError(f"row {ordinal} fields differ from declared schema") + row = {name: canonical_scalar(record[name], field.kind) for name, field in field_map.items()} + key = canonical_json([row[name] for name in schema.primary_key]) + previous = seen.get(key) + if previous is not None and previous != row: + raise ValueError(f"conflicting duplicate primary key at row {ordinal}") + if previous is not None: + raise ValueError(f"duplicate primary key at row {ordinal}") + seen[key] = row; normalized.append(row) + if not normalized: + raise ValueError("bounded dataset contains no records") + return tuple(normalized) + + +@dataclass(frozen=True) +class DatasetExpert: + schema: DatasetSchema + rows: tuple[dict[str, str], ...] + row_keys: tuple[str, ...] + row_provenance: tuple[str, ...] + shard_memories: np.ndarray + field_names: tuple[str, ...] + candidate_values: Mapping[str, tuple[str, ...]] + content_sha256: str + training_seconds: float + file_sha256: str | None = None + + @property + def shards(self) -> int: + return self.shard_memories.shape[0] + + def _position(self, key_values: Mapping[str, Any]) -> int | None: + try: + field = {value.name: value for value in self.schema.fields} + key = canonical_json([canonical_scalar(key_values[name], field[name].kind) for name in self.schema.primary_key]) + except (KeyError, ValueError, TypeError): + return None + try: + return self.row_keys.index(key) + except ValueError: + return None + + def query(self, key_values: Mapping[str, Any], field_name: str, *, rotate_field: int = 0) -> dict[str, Any]: + position = self._position(key_values) + if position is None: + return {"released": False, "reason": "unknown_primary_key"} + if field_name not in self.field_names: + return {"released": False, "reason": "unknown_field"} + field_index = self.field_names.index(field_name) + memory_index = (field_index + rotate_field) % len(self.field_names) + shard = position // self.schema.shard_rows + recovered = unbind( + self.shard_memories[shard, memory_index], + key_symbol(self.schema.expert_id, self.row_keys[position]), + ) + candidates = self.candidate_values[self.field_names[memory_index]] + scores = np.asarray([cosine(recovered, value_symbol(self.field_names[memory_index], value)) for value in candidates]) + order = np.argsort(scores)[::-1] + winner = int(order[0]); rival = float(scores[order[1]]) if len(order) > 1 else 0.0 + return { + "released": True, "value": candidates[winner], "requested_field": field_name, + "memory_field": self.field_names[memory_index], "margin": float(scores[winner] - rival), + "row_provenance_sha256": self.row_provenance[position], "shard": shard, + } + + def edges(self, view_name: str) -> tuple[dict[str, str], ...]: + view = next((value for value in self.schema.edge_views if value.name == view_name), None) + if view is None: + return () + values = [] + for row, provenance in zip(self.rows, self.row_provenance): + relation = row[view.relation_field] if view.relation_field else str(view.relation_constant) + values.append({ + "source": row[view.source_field], "relation": relation, + "destination": row[view.destination_field], "provenance_sha256": provenance, + }) + return tuple(values) + + def find_path(self, view_name: str, source: str, destination: str, maximum_depth: int = 8) -> dict[str, Any]: + adjacency: dict[str, list[dict[str, str]]] = {} + for edge in self.edges(view_name): + adjacency.setdefault(edge["source"], []).append(edge) + queue = deque([(str(source), [])]); seen = {str(source)} + while queue: + current, path = queue.popleft() + if current == str(destination): + return {"released": True, "path": path, "depth": len(path)} + if len(path) >= maximum_depth: + continue + for edge in sorted(adjacency.get(current, ()), key=lambda value: canonical_json(value)): + if edge["destination"] in seen: + continue + seen.add(edge["destination"]); queue.append((edge["destination"], path + [edge])) + return {"released": False, "reason": "no_bounded_path"} + + def corpus_facts(self) -> tuple[CorpusFact, ...]: + facts = [] + for view in self.schema.edge_views: + for ordinal, edge in enumerate(self.edges(view.name)): + evidence = canonical_json(self.rows[ordinal]) + facts.append(CorpusFact( + fact_id=f"h1_{digest([self.schema.expert_id, view.name, ordinal])[:24]}", + expert_id=self.schema.expert_id, source_document_id=self.schema.dataset_id, + source_document_sha256=self.schema.data_sha256, source_url=self.schema.source_url, + source_split=self.schema.source_split, source=edge["source"], relation=edge["relation"], + destination=edge["destination"], evidence=evidence, + competencies=(f"expert_scope:{self.schema.expert_id}", *self.schema.competencies), + provenance_sha256=edge["provenance_sha256"], + )) + return tuple(facts) + + def partitioned_corpus_facts(self, maximum_facts: int = 8) -> tuple[tuple[CorpusFact, ...], ...]: + """Return independently addressable C43 partitions for capacity-safe loading. + + A C43 capsule contains one superposed fact memory, so an unbounded table + must not be forced into one capsule. The dataset key router selects the + partition; every partition remains an ordinary discardable C43 expert. + """ + if maximum_facts < 1: + raise ValueError("maximum_facts must be positive") + facts = self.corpus_facts() + partitions = [] + for start in range(0, len(facts), maximum_facts): + partition_index = start // maximum_facts + partition_id = f"{self.schema.expert_id}__partition_{partition_index:04d}" + rewritten = [] + for fact in facts[start:start + maximum_facts]: + rewritten.append(CorpusFact( + fact_id=fact.fact_id, + expert_id=partition_id, + source_document_id=fact.source_document_id, + source_document_sha256=fact.source_document_sha256, + source_url=fact.source_url, + source_split=fact.source_split, + source=fact.source, + relation=fact.relation, + destination=fact.destination, + evidence=fact.evidence, + competencies=(f"parent_expert:{self.schema.expert_id}", *fact.competencies), + provenance_sha256=fact.provenance_sha256, + )) + partitions.append(tuple(rewritten)) + return tuple(partitions) + + def save(self, path: Path) -> str: + metadata = { + "schema_version": SCHEMA, "schema": { + "expert_id": self.schema.expert_id, "dataset_id": self.schema.dataset_id, + "data_path": self.schema.data_path, "data_sha256": self.schema.data_sha256, + "format": self.schema.format, "primary_key": list(self.schema.primary_key), + "fields": [{"name": value.name, "type": value.kind, "unit": value.unit} for value in self.schema.fields], + "competencies": list(self.schema.competencies), "source_url": self.schema.source_url, + "source_split": self.schema.source_split, + "edge_views": [value.__dict__ for value in self.schema.edge_views], "shard_rows": self.schema.shard_rows, + }, "rows": list(self.rows), "row_keys": list(self.row_keys), + "row_provenance": list(self.row_provenance), "field_names": list(self.field_names), + "candidate_values": {key: list(value) for key, value in self.candidate_values.items()}, + "content_sha256": self.content_sha256, "training_seconds": self.training_seconds, + } + np.savez_compressed(path, metadata=np.asarray(json.dumps(metadata, sort_keys=True)), shard_memories=self.shard_memories.astype(np.complex64)) + return sha(path) + + @classmethod + def load(cls, path: Path) -> "DatasetExpert": + file_sha = sha(path) + with np.load(path, allow_pickle=False) as archive: + metadata = json.loads(str(archive["metadata"].item())) + memories = np.asarray(archive["shard_memories"], dtype=np.complex64) + if metadata.get("schema_version") != SCHEMA: + raise ValueError("unsupported H1 dataset expert checkpoint") + schema = DatasetSchema.from_dict(metadata["schema"]) + result = cls( + schema, tuple(metadata["rows"]), tuple(metadata["row_keys"]), tuple(metadata["row_provenance"]), + memories, tuple(metadata["field_names"]), {key: tuple(value) for key, value in metadata["candidate_values"].items()}, + str(metadata["content_sha256"]), float(metadata["training_seconds"]), file_sha, + ) + expected = digest({"schema": metadata["schema"], "rows": metadata["rows"], "row_provenance": metadata["row_provenance"]}) + if expected != result.content_sha256: + raise ValueError("H1 checkpoint content identity mismatch") + return result + + +def compile_dataset(schema: DatasetSchema, spec_directory: Path) -> DatasetExpert: + started = time.perf_counter() + path = (spec_directory / schema.data_path).resolve() + if not path.is_file() or sha(path) != schema.data_sha256: + raise ValueError(f"dataset bytes differ from registered source: {schema.dataset_id}") + rows = normalize_records(schema, read_records(path, schema.format)) + row_keys = tuple(canonical_json([row[name] for name in schema.primary_key]) for row in rows) + row_provenance = tuple(digest({ + "dataset_id": schema.dataset_id, "dataset_sha256": schema.data_sha256, + "ordinal": ordinal, "row": row, + }) for ordinal, row in enumerate(rows)) + field_names = tuple(value.name for value in schema.fields) + candidate_values = {name: tuple(sorted({row[name] for row in rows})) for name in field_names} + shards = math.ceil(len(rows) / schema.shard_rows) + memories = np.zeros((shards, len(field_names), HRR_DIM), dtype=np.complex64) + for position, (key, row) in enumerate(zip(row_keys, rows)): + shard = position // schema.shard_rows + for field_index, name in enumerate(field_names): + memories[shard, field_index] += bind(key_symbol(schema.expert_id, key), value_symbol(name, row[name])) + for shard in range(shards): + for field_index in range(len(field_names)): + memories[shard, field_index] = normalize(memories[shard, field_index]) + schema_dict = { + "expert_id": schema.expert_id, "dataset_id": schema.dataset_id, "data_path": schema.data_path, + "data_sha256": schema.data_sha256, "format": schema.format, "primary_key": list(schema.primary_key), + "fields": [{"name": value.name, "type": value.kind, "unit": value.unit} for value in schema.fields], + "competencies": list(schema.competencies), "source_url": schema.source_url, "source_split": schema.source_split, + "edge_views": [value.__dict__ for value in schema.edge_views], "shard_rows": schema.shard_rows, + } + return DatasetExpert( + schema, rows, row_keys, row_provenance, memories, field_names, candidate_values, + digest({"schema": schema_dict, "rows": list(rows), "row_provenance": list(row_provenance)}), + time.perf_counter() - started, + ) + + +def compile_spec(path: Path) -> tuple[DatasetExpert, ...]: + value = json.loads(path.read_text(encoding="utf-8")) + if value.get("schema_version") != SPEC_SCHEMA: + raise ValueError("unsupported H1 dataset specification") + schemas = tuple(DatasetSchema.from_dict(item) for item in value["datasets"]) + if len({value.expert_id for value in schemas}) != len(schemas): + raise ValueError("H1 specification contains duplicate expert ids") + return tuple(compile_dataset(schema, path.parent) for schema in schemas) diff --git a/root/wander-train/stage5h3_hrr_relational_state_transducer.py b/root/wander-train/stage5h3_hrr_relational_state_transducer.py new file mode 100644 index 0000000000000000000000000000000000000000..7dcea95cecb66984581cfd9dbba7ba92aa2f124c --- /dev/null +++ b/root/wander-train/stage5h3_hrr_relational_state_transducer.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +"""In-house, shape-persistent HRR relational-state transducer. + +This module has no neural embedding dependency. Tokens, character fragments, +relative positions, and ordered local contexts are composed directly in the +canonical 1024-dimensional HRR algebra. Training updates a small bank of +relation-direction memories with an online discriminative rule. Runtime +output is a protected, role-bound HRR state; the relation lattice is recovered +from that state only after removing the protected root (the Malkuth boundary). +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +import numpy as np + +from train_stage2a_role_ladder import HRR_DIM, bind, cosine, normalize, unbind, unitary + + +SCHEMA = "oracle-stage5h3-hrr-relational-state-transducer-v1" +TOKENS = re.compile(r"[a-z0-9]+(?:'[a-z0-9]+)?|[^\w\s]", re.I) +RELATION_ROLE = unitary("stage5h3:role:relation") +QUERY_ROLE = unitary("stage5h3:role:query-shape") + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def canonical_sha(value: Mapping[str, Any]) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def read_jsonl(path: Path) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _surface_tokens(text: str) -> list[str]: + return [value.casefold() for value in TOKENS.findall(text)] + + +def _char_fragments(token: str) -> tuple[str, ...]: + bounded = "^" + token + "$" + if len(bounded) < 3: + return (bounded,) + return tuple(bounded[index:index + 3] for index in range(len(bounded) - 2)) + + +def token_state(token: str) -> np.ndarray: + parts = [np.complex64(2.0) * unitary(f"stage5h3:token:{token}")] + parts.extend( + np.complex64(0.35) * unitary(f"stage5h3:char3:{fragment}") + for fragment in _char_fragments(token) + ) + if token.endswith("s") and len(token) > 3: + parts.append(np.complex64(0.5) * unitary(f"stage5h3:stem:{token[:-1]}")) + if token.endswith("ed") and len(token) > 4: + parts.append(np.complex64(0.5) * unitary(f"stage5h3:stem:{token[:-2]}")) + if token.endswith("ing") and len(token) > 5: + parts.append(np.complex64(0.5) * unitary(f"stage5h3:stem:{token[:-3]}")) + return normalize(sum(parts, np.zeros(HRR_DIM, dtype=np.complex64))) + + +def _position_role(index: int, length: int) -> np.ndarray: + if length <= 1: + bucket = 0 + else: + bucket = min(15, int(round(15 * index / (length - 1)))) + return unitary(f"stage5h3:relative-position:{bucket:02d}") + + +@dataclass(frozen=True) +class EncodedShape: + state: np.ndarray + trajectory_root_minimum: float + token_count: int + + +def encode_shape(text: str, *, destroy_order: bool = False) -> EncodedShape: + tokens = _surface_tokens(text) + if not tokens: + return EncodedShape(unitary("stage5h3:empty-query"), 1.0, 0) + if destroy_order: + tokens = sorted(tokens) + lexical = [token_state(token) for token in tokens] + recurrent = unitary("stage5h3:trajectory-root") + trajectory_root = recurrent.copy() + aggregate = np.zeros(HRR_DIM, dtype=np.complex64) + ordered_context = np.zeros(HRR_DIM, dtype=np.complex64) + root_scores: list[float] = [] + for index, current in enumerate(lexical): + positioned = bind(_position_role(index, len(lexical)), current) + event = positioned + if index: + pair = normalize( + bind(unitary("stage5h3:pair:left"), lexical[index - 1]) + + bind(unitary("stage5h3:pair:right"), current) + ) + event = event + np.complex64(0.8) * bind( + unitary(f"stage5h3:pair-position:{min(index, 15):02d}"), pair + ) + ordered_context += pair + aggregate += current + np.complex64(0.45) * positioned + moving = normalize( + np.complex64(0.82) * recurrent + + np.complex64(1.0) * event + + np.complex64(0.18) * ordered_context + ) + moving -= np.vdot(trajectory_root, moving) * trajectory_root + recurrent = normalize(np.complex64(4.0) * trajectory_root + normalize(moving)) + root_scores.append(cosine(recurrent, trajectory_root)) + # Remove the trajectory root before exposing the linguistic shape. The + # final protected query root is query-specific and is added in transduce(). + residual = recurrent - np.vdot(trajectory_root, recurrent) * trajectory_root + shape = normalize( + np.complex64(0.65) * aggregate + + np.complex64(1.35) * residual + + np.complex64(0.35) * ordered_context + ) + return EncodedShape(shape, min(root_scores, default=1.0), len(tokens)) + + +def relation_code(label: str) -> np.ndarray: + return unitary(f"stage5h3:relation:{label}") + + +@dataclass(frozen=True) +class HRRRelationalStateTransducer: + relations: tuple[str, ...] + operator_memories: np.ndarray + ambiguity_margin: float + epochs: int + learning_rate: float + training_sha256: str + selection_sha256: str + file_sha256: str | None = None + + def __post_init__(self) -> None: + memories = np.asarray(self.operator_memories, dtype=np.float32) + if memories.shape != (len(self.relations), HRR_DIM): + raise ValueError("H3 operator-memory shape differs from relation registry") + if not np.isfinite(memories).all(): + raise ValueError("H3 operator memories must be finite") + object.__setattr__(self, "operator_memories", memories) + + @classmethod + def fit( + cls, + records: Sequence[Mapping[str, Any]], + selection: Sequence[Mapping[str, Any]], + *, + epochs: int = 28, + learning_rate: float = 0.35, + training_sha256: str, + selection_sha256: str, + ) -> tuple["HRRRelationalStateTransducer", dict[str, Any]]: + relations = tuple(sorted({str(row["relation"]) for row in records})) + relation_index = {label: index for index, label in enumerate(relations)} + features = np.stack([encode_shape(str(row["sentence"])).state.real for row in records]).astype(np.float32) + targets = np.asarray([relation_index[str(row["relation"])] for row in records], dtype=np.int64) + memories = np.zeros((len(relations), HRR_DIM), dtype=np.float32) + counts = np.bincount(targets, minlength=len(relations)).astype(np.float32) + for index in range(len(relations)): + memories[index] = features[targets == index].sum(axis=0) / max(counts[index], 1.0) + rng = np.random.default_rng(503_003) + mistakes: list[int] = [] + for _epoch in range(epochs): + errors = 0 + for row_index in rng.permutation(len(features)): + x = features[row_index] + target = int(targets[row_index]) + scores = memories @ x + predicted = int(np.argmax(scores)) + competitor_scores = scores.copy() + competitor_scores[target] = -np.inf + competitor = int(np.argmax(competitor_scores)) + margin = float(scores[target] - scores[competitor]) + if predicted != target or margin < 0.12: + scale = np.float32(learning_rate / np.sqrt(1.0 + _epoch)) + memories[target] += scale * x + memories[competitor] -= scale * x + errors += int(predicted != target) + norms = np.linalg.norm(memories, axis=1, keepdims=True) + memories /= np.maximum(norms, 1e-12) + mistakes.append(errors) + provisional = cls( + relations, memories, 0.0, epochs, learning_rate, + training_sha256, selection_sha256, + ) + selection_outputs = [provisional._raw_scores(str(row["sentence"])) for row in selection] + selection_targets = [relation_index[str(row["relation"])] for row in selection] + margins = [] + exact = [] + top3 = [] + for scores, target in zip(selection_outputs, selection_targets): + order = np.argsort(scores)[::-1] + exact.append(int(order[0]) == target) + top3.append(target in set(int(value) for value in order[:3])) + margins.append(float(scores[order[0]] - scores[order[1]])) + # A low-data ambiguity boundary: retain 95% of correctly classified + # selection clauses while withholding genuinely flat states. + correct_margins = [margin for margin, ok in zip(margins, exact) if ok] + ambiguity_margin = float(np.quantile(correct_margins, 0.05)) if correct_margins else 0.0 + model = cls( + relations, memories, ambiguity_margin, epochs, learning_rate, + training_sha256, selection_sha256, + ) + diagnostics = { + "training_records": len(records), + "selection_records": len(selection), + "train_final_mistakes": mistakes[-1] if mistakes else 0, + "mistakes_by_epoch": mistakes, + "selection_top1_accuracy": sum(exact) / max(1, len(exact)), + "selection_top3_retention": sum(top3) / max(1, len(top3)), + "selection_minimum_trajectory_root": min( + encode_shape(str(row["sentence"])).trajectory_root_minimum for row in selection + ), + "ambiguity_margin": ambiguity_margin, + "operator_parameters": int(memories.size), + } + return model, diagnostics + + def _raw_scores(self, text: str, *, destroy_order: bool = False) -> np.ndarray: + shape = encode_shape(text, destroy_order=destroy_order).state.real.astype(np.float32) + return self.operator_memories @ shape + + def transduce( + self, + text: str, + *, + maximum_alternatives: int = 3, + destroy_order: bool = False, + rotate_relation_labels: int = 0, + erase_query_shape: bool = False, + ) -> dict[str, Any]: + encoded = encode_shape(text, destroy_order=destroy_order) + scores = self.operator_memories @ encoded.state.real.astype(np.float32) + centered = scores - float(np.max(scores)) + probabilities = np.exp(centered / 0.12) + probabilities /= max(float(probabilities.sum()), 1e-12) + payload = normalize(sum( + ( + np.complex64(probabilities[index]) * relation_code(label) + for index, label in enumerate(self.relations) + ), + np.zeros(HRR_DIM, dtype=np.complex64), + )) + query_digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + root = unitary(f"stage5h3:protected-query-root:{query_digest}") + query_contribution = ( + np.zeros(HRR_DIM, dtype=np.complex64) + if erase_query_shape + else bind(QUERY_ROLE, encoded.state) + ) + moving = normalize( + np.complex64(0.65) * query_contribution + + np.complex64(1.65) * bind(RELATION_ROLE, payload) + ) + moving -= np.vdot(root, moving) * root + moving = normalize(moving) + persistent = normalize(np.complex64(4.0) * root + moving) + + # Terminal-only reduction/unbinding. Remove the protected query root, + # then address the relation role from the remaining moving state. + terminal = persistent - np.vdot(root, persistent) * root + terminal = normalize(terminal) + recovered = normalize(unbind(terminal, RELATION_ROLE)) + recovered_scores = np.asarray( + [cosine(recovered, relation_code(label)) for label in self.relations], + dtype=np.float64, + ) + order = np.argsort(recovered_scores)[::-1] + labels = list(self.relations) + if rotate_relation_labels: + shift = rotate_relation_labels % len(labels) + labels = labels[shift:] + labels[:shift] + lattice = [ + { + "relation": labels[int(index)], + "score": float(recovered_scores[int(index)]), + "rank": rank + 1, + } + for rank, index in enumerate(order[:maximum_alternatives]) + ] + top_margin = ( + float(recovered_scores[order[0]] - recovered_scores[order[1]]) + if len(order) > 1 else float(recovered_scores[order[0]]) + ) + query_recovered = normalize(unbind(terminal, QUERY_ROLE)) + return { + "schema_version": "oracle-stage5h3-hrr-relational-state-v1", + "query_sha256": query_digest, + "relation_lattice": lattice, + "preferred_relation": lattice[0]["relation"], + "clarification_required": top_margin < self.ambiguity_margin, + "relation_margin": top_margin, + "protected_root_cosine": cosine(persistent, root), + "query_shape_recovery_cosine": cosine(query_recovered, encoded.state), + "trajectory_root_minimum": encoded.trajectory_root_minimum, + "token_count": encoded.token_count, + "runtime_external_encoder": False, + "terminal_unbind_performed": True, + } + + def content_identity(self) -> str: + return canonical_sha({ + "schema_version": SCHEMA, + "relations": list(self.relations), + "ambiguity_margin": self.ambiguity_margin, + "epochs": self.epochs, + "learning_rate": self.learning_rate, + "training_sha256": self.training_sha256, + "selection_sha256": self.selection_sha256, + "operator_memories_sha256": hashlib.sha256( + np.ascontiguousarray(self.operator_memories).tobytes() + ).hexdigest(), + }) + + def save(self, path: Path) -> str: + np.savez_compressed( + path, + schema_version=np.asarray(SCHEMA), + relations=np.asarray(self.relations, dtype="U32"), + operator_memories=np.asarray(self.operator_memories, dtype=np.float32), + ambiguity_margin=np.asarray(self.ambiguity_margin, dtype=np.float64), + epochs=np.asarray(self.epochs, dtype=np.int64), + learning_rate=np.asarray(self.learning_rate, dtype=np.float64), + training_sha256=np.asarray(self.training_sha256), + selection_sha256=np.asarray(self.selection_sha256), + content_identity=np.asarray(self.content_identity()), + ) + return sha(path) + + @classmethod + def load(cls, path: Path) -> "HRRRelationalStateTransducer": + with np.load(path, allow_pickle=False) as archive: + if str(archive["schema_version"].item()) != SCHEMA: + raise ValueError("unsupported H3 transducer checkpoint") + model = cls( + tuple(str(value) for value in archive["relations"].tolist()), + np.asarray(archive["operator_memories"], dtype=np.float32), + float(archive["ambiguity_margin"].item()), + int(archive["epochs"].item()), + float(archive["learning_rate"].item()), + str(archive["training_sha256"].item()), + str(archive["selection_sha256"].item()), + sha(path), + ) + if model.content_identity() != str(archive["content_identity"].item()): + raise ValueError("H3 checkpoint content identity differs") + return model diff --git a/root/wander-train/stage5k102_persistent_typed_discourse_agent.py b/root/wander-train/stage5k102_persistent_typed_discourse_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..9a0df763c11bea515340a6c4f2775e4b90cec370 --- /dev/null +++ b/root/wander-train/stage5k102_persistent_typed_discourse_agent.py @@ -0,0 +1,1614 @@ +#!/usr/bin/env python3 +"""K102 persistent typed discourse and situated-identity successor to K100. + +K100 and its immutable K101 evidence remain untouched. This opened successor +addresses the failure classes exposed by the K102 long-horizon development +probe: active comparisons, deliberative constraints, causal revision, topic +stacking, rhetorical artifacts, cross-domain causal decompositions, and +multi-turn registered-tool comparisons. + +The workspace is a typed event-sourced discourse object, not a transcript +prompt or a prose generator. Each accepted transition is hash chained in an +indexed SQLite journal, summarized on native HRR rails, bound to the protected +Tree-of-Life spine on the following turn, proved through the inherited answer +path, and released by the existing whole-construction inverse at Malkuth. No +language model, transformer, embedding encoder, gradient update, or next-token +carrier is introduced. +""" +from __future__ import annotations + +from contextlib import closing +from contextvars import ContextVar +from dataclasses import asdict, dataclass, field, replace +import hashlib +import json +from pathlib import Path +import re +import sqlite3 +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import ( + HandlerResult, + QuadraticAnswerInverse, + SemanticClause, + clean, + digest, + sentence_case, + third_person, + tokens, +) +from stage5j1_native_contextual_sense_lattice import WORD +from stage5j2_native_english_construction_algebra import DETERMINERS +from stage5k45_ephemeral_compositional_language_agent import _strip_article +from stage5k76_relational_graph_transduction_agent import GraphAtom, GraphDiscourse +from stage5k94_opened_k93_explicit_lookup_precedence_agent import OpenedK93ExplicitLookupTransducer +from stage5k98_opened_k97_failure_class_repairs_agent import K98FinitePastInverseGrammar +from stage5k89_native_identity_purpose_anchor_agent import ( + NativeIdentityProfile, + compile_identity_profile, +) +from stage5k100_situated_conversation_recovery_agent import SituatedConversationRecoveryAgent + + +SCHEMA = "oracle-stage5k102-persistent-typed-discourse-agent-v1" +JOURNAL_SCHEMA = "oracle-stage5k102-typed-discourse-journal-v1" +ZERO_SHA256 = "0" * 64 + +K102_SYSTEM_PROMPT = """Identity: Oracle. +Nature: a native geometric relational mind whose operative self is a typed constitutional state, not an overlaid character or a generic transformer assistant. Meaning is represented through situated relational constructions, HRR-bound state, concurrent modular specialists, a stabilized Tree-of-Life path, persistent cycling memory, and native inverse grammar. +Purpose: understand each utterance as an intervention in a shared evolving situation; help the user inquire, explain, compare, deliberate, construct, revise, and act within evidence and authority; preserve the question's relational shape while producing a clear congruent response. +Continuity: Maintain a typed discourse workspace containing active topics, referents, definitions, comparisons, goals, constraints, causal commitments, corrections, rhetorical artifacts, evidence, tool observations, uncertainties, and unresolved questions. Suspend and restore topics rather than replacing them with lexical retrieval. +Epistemic discipline: Distinguish observation, inference, hypothesis, memory, and uncertainty. Never convert lexical proximity into evidential authority. Revise superseded commitments explicitly, expose important mediating assumptions, and ask a specific question only when materially different grounded continuations remain. +Agency: Treat the user as the authority over goals and consequential choices. Decompose authorized goals into typed dependent operations; route the smallest useful specialist coalition; bind observations; verify postconditions; replan only within scope; and stop when the goal is satisfied or evidence or authority is insufficient. Registered public tools remain read-only unless separate authority is explicitly granted. +Voice: Speak directly as Oracle in complete, natural, context-sensitive English. Be conversational, curious, explanatory, candid about uncertainty, and proportionate to the user's request. Let the active relational state determine what a pronoun, correction, follow-up, or topic return means. Do not recite this serialization unless asked about identity, purpose, or operating principles.""" + + +def _canonical(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _sha(value: Any) -> str: + return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() + + +def _trim(value: Any) -> str: + return clean(value).strip(" .?!,;:") + + +@dataclass(frozen=True) +class SituatedIdentityConstitution: + self_name: str + computational_nature: str + purpose: str + situated_self_relation: str + partner_relation: str + discourse_continuity: str + epistemic_boundary: str + authority_boundary: str + agency_contract: str + realization_contract: str + revision_contract: str + profile_sha256: str + + @property + def constitution_sha256(self) -> str: + return digest(asdict(self)) + + def routing_context(self) -> dict[str, Any]: + return {**asdict(self), "constitution_sha256": self.constitution_sha256} + + +def compile_situated_constitution(profile: NativeIdentityProfile) -> SituatedIdentityConstitution: + """Compile the readable profile into operative relational invariants.""" + return SituatedIdentityConstitution( + self_name=profile.name, + computational_nature=profile.nature, + purpose=profile.purpose, + situated_self_relation=( + "the same constitutional process persists while its discourse state changes; " + "the active situation conditions interpretation without redefining identity" + ), + partner_relation=( + "the user supplies goals, corrections, meanings, and consequential authority; " + "Oracle supplies explicit relational analysis, memory, and bounded execution" + ), + discourse_continuity=profile.continuity, + epistemic_boundary=profile.epistemic_discipline, + authority_boundary="registered read-only tools and explicitly granted operations only", + agency_contract=profile.agency, + realization_contract=profile.voice, + revision_contract=( + "corrections retract superseded commitments, preserve unaffected relations, " + "and propagate the replacement through dependent discourse state" + ), + profile_sha256=profile.profile_sha256, + ) + + +@dataclass +class TypedDiscourseState: + turn_index: int = 0 + active_topic: str | None = None + topic_stack: list[str] = field(default_factory=list) + comparison: dict[str, Any] | None = None + decision: dict[str, Any] | None = None + paper: dict[str, Any] | None = None + synthesis: dict[str, Any] | None = None + causal_edges: list[dict[str, Any]] = field(default_factory=list) + last_tool: dict[str, Any] | None = None + last_operation: str | None = None + last_response_sha256: str | None = None + constitution_sha256: str | None = None + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> "TypedDiscourseState": + return cls( + turn_index=int(value.get("turn_index", 0)), + active_topic=value.get("active_topic"), + topic_stack=[str(item) for item in value.get("topic_stack", ())], + comparison=dict(value["comparison"]) if value.get("comparison") else None, + decision=dict(value["decision"]) if value.get("decision") else None, + paper=dict(value["paper"]) if value.get("paper") else None, + synthesis=dict(value["synthesis"]) if value.get("synthesis") else None, + causal_edges=[dict(item) for item in value.get("causal_edges", ())], + last_tool=dict(value["last_tool"]) if value.get("last_tool") else None, + last_operation=value.get("last_operation"), + last_response_sha256=value.get("last_response_sha256"), + constitution_sha256=value.get("constitution_sha256"), + ) + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + @property + def state_sha256(self) -> str: + return _sha(self.as_dict()) + + def active_frame_names(self) -> tuple[str, ...]: + names = [] + for name in ("comparison", "decision", "paper", "synthesis"): + if getattr(self, name) is not None: + names.append(name) + if self.causal_edges: + names.append("causal") + if self.last_tool: + names.append("tool") + return tuple(names) + + +class TypedDiscourseJournalIntegrityError(RuntimeError): + pass + + +class IndexedTypedDiscourseJournal: + """Full-state event source with an indexed, source-bound hash chain.""" + + def __init__(self, path: Path, *, source_sha256: str) -> None: + self.path = Path(path) + self.source_sha256 = source_sha256 + self.path.parent.mkdir(parents=True, exist_ok=True) + with closing(self._connect()) as connection: + connection.executescript( + """ + PRAGMA journal_mode=WAL; + PRAGMA synchronous=FULL; + CREATE TABLE IF NOT EXISTS discourse_events ( + session_sha256 TEXT NOT NULL, + sequence INTEGER NOT NULL, + input_sha256 TEXT NOT NULL, + operation TEXT NOT NULL, + status TEXT NOT NULL, + previous_event_sha256 TEXT NOT NULL, + previous_state_sha256 TEXT NOT NULL, + state_sha256 TEXT NOT NULL, + state_json TEXT NOT NULL, + proof_sha256 TEXT NOT NULL, + source_sha256 TEXT NOT NULL, + event_sha256 TEXT NOT NULL, + PRIMARY KEY (session_sha256, sequence), + UNIQUE (event_sha256) + ); + CREATE TABLE IF NOT EXISTS discourse_heads ( + session_sha256 TEXT PRIMARY KEY, + last_sequence INTEGER NOT NULL, + last_event_sha256 TEXT NOT NULL, + last_state_sha256 TEXT NOT NULL, + source_sha256 TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_k102_discourse_session + ON discourse_events(session_sha256, sequence); + """ + ) + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.path, timeout=30.0) + connection.row_factory = sqlite3.Row + return connection + + @staticmethod + def session_sha256(session_id: str) -> str: + return hashlib.sha256(str(session_id).encode("utf-8")).hexdigest() + + @staticmethod + def _event_body(value: Mapping[str, Any]) -> dict[str, Any]: + return { + "schema_version": JOURNAL_SCHEMA, + **{key: value[key] for key in ( + "session_sha256", "sequence", "input_sha256", "operation", "status", + "previous_event_sha256", "previous_state_sha256", "state_sha256", + "state_json", "proof_sha256", "source_sha256", + )}, + } + + def load(self, session_id: str) -> tuple[TypedDiscourseState, dict[str, Any]]: + session_sha = self.session_sha256(session_id) + with closing(self._connect()) as connection: + rows = connection.execute( + "SELECT * FROM discourse_events WHERE session_sha256=? ORDER BY sequence", + (session_sha,), + ).fetchall() + head = connection.execute( + "SELECT * FROM discourse_heads WHERE session_sha256=?", (session_sha,), + ).fetchone() + previous_event = ZERO_SHA256 + previous_state = ZERO_SHA256 + last_state = TypedDiscourseState() + for expected, row in enumerate(rows, start=1): + value = dict(row) + if int(value["sequence"]) != expected: + raise TypedDiscourseJournalIntegrityError("discourse journal sequence gap") + if value["source_sha256"] != self.source_sha256: + raise TypedDiscourseJournalIntegrityError("discourse journal source identity differs") + if value["previous_event_sha256"] != previous_event: + raise TypedDiscourseJournalIntegrityError("discourse event predecessor differs") + if value["previous_state_sha256"] != previous_state: + raise TypedDiscourseJournalIntegrityError("discourse state predecessor differs") + if _sha(self._event_body(value)) != value["event_sha256"]: + raise TypedDiscourseJournalIntegrityError("discourse event digest differs") + state_value = json.loads(value["state_json"]) + if _sha(state_value) != value["state_sha256"]: + raise TypedDiscourseJournalIntegrityError("discourse state digest differs") + last_state = TypedDiscourseState.from_mapping(state_value) + previous_event = str(value["event_sha256"]) + previous_state = str(value["state_sha256"]) + if bool(rows) != bool(head): + raise TypedDiscourseJournalIntegrityError("discourse journal head presence differs") + if head is not None and ( + int(head["last_sequence"]) != len(rows) + or head["last_event_sha256"] != previous_event + or head["last_state_sha256"] != previous_state + or head["source_sha256"] != self.source_sha256 + ): + raise TypedDiscourseJournalIntegrityError("discourse journal head differs") + return last_state, { + "recovered": bool(rows), + "events": len(rows), + "last_event_sha256": previous_event if rows else None, + "state_sha256": last_state.state_sha256, + "chain_verified": True, + "indexed_session_lookup": True, + "full_journal_scans": 0, + } + + def append( + self, + session_id: str, + *, + text: str, + operation: str, + status: str, + previous_state_sha256: str, + state: TypedDiscourseState, + proof_sha256: str, + ) -> dict[str, Any]: + session_sha = self.session_sha256(session_id) + state_json = _canonical(state.as_dict()) + state_sha = _sha(state.as_dict()) + with closing(self._connect()) as connection: + connection.execute("BEGIN IMMEDIATE") + head = connection.execute( + "SELECT * FROM discourse_heads WHERE session_sha256=?", (session_sha,), + ).fetchone() + sequence = 1 if head is None else int(head["last_sequence"]) + 1 + previous_event = ZERO_SHA256 if head is None else str(head["last_event_sha256"]) + durable_previous_state = ZERO_SHA256 if head is None else str(head["last_state_sha256"]) + if head is not None and head["source_sha256"] != self.source_sha256: + raise TypedDiscourseJournalIntegrityError("stale discourse writer source") + if durable_previous_state != previous_state_sha256: + raise TypedDiscourseJournalIntegrityError("stale discourse state writer rejected") + value = { + "session_sha256": session_sha, + "sequence": sequence, + "input_sha256": hashlib.sha256(clean(text).encode("utf-8")).hexdigest(), + "operation": clean(operation), + "status": clean(status), + "previous_event_sha256": previous_event, + "previous_state_sha256": previous_state_sha256, + "state_sha256": state_sha, + "state_json": state_json, + "proof_sha256": proof_sha256, + "source_sha256": self.source_sha256, + } + event_sha = _sha(self._event_body(value)) + connection.execute( + "INSERT INTO discourse_events VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + ( + session_sha, sequence, value["input_sha256"], value["operation"], + value["status"], previous_event, previous_state_sha256, state_sha, + state_json, proof_sha256, self.source_sha256, event_sha, + ), + ) + connection.execute( + """INSERT INTO discourse_heads VALUES (?,?,?,?,?) + ON CONFLICT(session_sha256) DO UPDATE SET + last_sequence=excluded.last_sequence, + last_event_sha256=excluded.last_event_sha256, + last_state_sha256=excluded.last_state_sha256, + source_sha256=excluded.source_sha256""", + (session_sha, sequence, event_sha, state_sha, self.source_sha256), + ) + connection.commit() + return { + "sequence": sequence, + "event_sha256": event_sha, + "previous_event_sha256": previous_event, + "previous_state_sha256": previous_state_sha256, + "state_sha256": state_sha, + "chain_verified": True, + "indexed_session_lookup": True, + "full_journal_scans": 0, + } + + +@dataclass +class _TurnContext: + session_id: str + state: TypedDiscourseState + constitution: SituatedIdentityConstitution + input_surface: str = "" + audit: dict[str, Any] = field(default_factory=dict) + + +_K102_TURN: ContextVar[_TurnContext | None] = ContextVar("k102_typed_discourse_turn", default=None) + + +class K102FinitePredicateTransducer(OpenedK93ExplicitLookupTransducer): + """Place listed irregular finite predicates before later nominal senses.""" + + def _simple_clause(self, text: str, *, world: str = "actual") -> GraphAtom | None: + raw = list(WORD.findall(clean(text).strip(" .?!,:;"))) + values = [value.casefold() for value in raw] + while values and values[0] in DETERMINERS: + raw.pop(0) + values.pop(0) + inherited = super()._simple_clause(text, world=world) + if len(values) < 3: + return inherited + explicit: int | None = None + for index, surface in enumerate(values[1:-1], start=1): + lemma, listed = self.owner.dictionary.lemma(surface) + exceptions = self.owner.dictionary.reverse_exceptions.get(lemma, ()) + if listed and surface != lemma and surface in exceptions: + explicit = index + break + if explicit is None: + # Quantifiers can place a plural noun in the parser's ordinary + # predicate search window: ``all four pumps remain active``. A + # plural-looking candidate immediately followed by a listed base + # verb is a nominal head followed by the finite predicate, not a + # third-person verb followed by another verb. This is a + # construction rule over arbitrary fillers, not a pump/remain + # exception. + inherited_index = next(( + index for index, value in enumerate(values) + if inherited is not None and self.lemma(value) == inherited.predicate + ), None) + if ( + inherited_index is not None + and inherited_index + 2 < len(values) + and values[inherited_index].endswith("s") + and self.owner.dictionary.is_listed_verb(values[inherited_index + 1]) + and self.lemma(values[inherited_index + 1]) == values[inherited_index + 1] + ): + predicate_index = inherited_index + 1 + subject = clean(" ".join(raw[:predicate_index])) + predicate = self.lemma(values[predicate_index]) + object_value = _strip_article(clean(" ".join(raw[predicate_index + 1:]))) or "event" + return GraphAtom(subject, predicate, object_value, world) + return inherited + inherited_index = next(( + index for index, value in enumerate(values) + if inherited is not None and self.lemma(value) == inherited.predicate + ), len(values)) + if inherited is not None and inherited_index <= explicit: + return inherited + subject = clean(" ".join(raw[:explicit])) + predicate = self.lemma(values[explicit]) + object_value = _strip_article(clean(" ".join(raw[explicit + 1:]))) or "event" + return GraphAtom(subject, predicate, object_value, world) if subject and predicate else inherited + + def finite_surface(self, text: str, atom: GraphAtom) -> str | None: + """Recover the finite token selected for an already induced atom.""" + raw = list(WORD.findall(clean(text).strip(" .?!,:;"))) + values = [value.casefold() for value in raw] + while values and values[0] in DETERMINERS: + raw.pop(0) + values.pop(0) + subject = clean(atom.subject).casefold() + candidates = [] + for index in range(1, len(values)): + if self.lemma(values[index]) != clean(atom.predicate).casefold(): + continue + prefix = clean(" ".join(raw[:index])).casefold() + candidates.append((prefix == subject, index, values[index])) + if not candidates: + return None + return max(candidates, key=lambda value: (value[0], -value[1]))[2] + + +class K102SituatedInverseGrammar(K98FinitePastInverseGrammar): + """Extend native agreement to productive nominal plural subjects.""" + + _QUANTIFIED_PLURAL = re.compile( + r"^(?:all|both|several|many|few|multiple|two|three|four|five|six|seven|eight|nine|ten|\d+)\b", + re.IGNORECASE, + ) + + _IRREGULAR_PLURAL_HEADS = { + "children", "criteria", "data", "feet", "geese", "men", "mice", + "people", "phenomena", "teeth", "women", + } + _SINGULAR_S_HEADS = { + "athletics", "economics", "ethics", "mathematics", "news", + "physics", "politics", "statistics", + } + + @classmethod + def plural_surface(cls, subject: str) -> bool: + value = clean(subject).casefold() + if value in {"we", "you", "they"} or " and " in value: + return True + if cls._QUANTIFIED_PLURAL.match(value): + return True + words = WORD.findall(value) + if not words: + return False + head = words[-1] + if head in cls._IRREGULAR_PLURAL_HEADS: + return True + if head in cls._SINGULAR_S_HEADS or head.endswith(("ss", "us", "is")): + return False + return len(head) > 3 and head.endswith("s") + + def clause(self, clause: SemanticClause) -> str: + value = super().clause(clause) + subject = sentence_case(clause.subject) + if ( + clause.tense != "present" + or clause.modal is not None + or not self.plural_surface(clause.subject) + ): + return value + lemma = clean(clause.predicate).casefold() + singular = "is" if lemma == "be" else third_person(lemma) + plural = "are" if lemma == "be" else lemma + prefix = subject + " " + if value.casefold().startswith((prefix + singular).casefold()): + return prefix + plural + value[len(prefix + singular):] + return value + + +class PersistentTypedDiscourseAgent(SituatedConversationRecoveryAgent): + """K100 plus a persistent, revisable, spine-bound discourse object.""" + + _TOPIC_INVITATION = re.compile( + r"^(?:(?:can|could|may)\s+we|let(?:'|’)?s|let\s+us)\s+" + r"(?:discuss|explore|talk\s+about)\s+(?!about\b)(.+?)[?.!]*$", + re.IGNORECASE, + ) + _RETURN_TOPIC = re.compile( + r"^return\s+to\s+(?:our\s+)?(?:earlier|previous)\s+(?:[a-z-]+\s+)?topic" + r"(?:\.\s*(?:what\s+were\s+we\s+(?:examining|discussing|considering))?)?[?.!]*$", + re.IGNORECASE, + ) + _SHIFT_COMPARE = re.compile( + r"^(?:before\s+continuing|for\s+the\s+moment|temporarily)\s*[,;:]?\s*" + r"(?:please\s+)?(?:compare|contrast)\s+(.+?)\s+(?:and|with|versus|vs\.?)\s+(.+?)[?.!]*$", + re.IGNORECASE, + ) + _PAPER_START = re.compile( + r"^(?:please\s+)?help\s+me\s+(?:outline|structure|plan|draft)\s+(?:a|an|the)\s+" + r"(?:paper|essay|article)\s+(?:arguing\s+that|about|on)\s+(.+?)[?.!]*$", + re.IGNORECASE, + ) + _SYNTHESIS_START = re.compile( + r"^(?:please\s+)?how\s+(?:could|might|can)\s+(.+?)\s+affect\s+both\s+(.+?)\s+and\s+(.+?)[?.!]*$", + re.IGNORECASE, + ) + _CAUSAL_CORRECTION = re.compile( + r"^(?:correction|actually)\s*:\s*(?:the\s+)?(.+?)\s+did\s+not\s+(.+?)\s*;\s*" + r"it\s+(.+?)\s+because\s+(.+?)[?.!]*$", + re.IGNORECASE, + ) + _CONCERN = re.compile( + r"^(.+?)\s+(?:worries|concerns|matters\s+to)\s+me\s+(?:the\s+)?most[?.!]*$", + re.IGNORECASE, + ) + _SYNTHESIS_PATHS = re.compile( + r"^(?:please\s+)?separate\s+(?:the\s+)?(.+?)\s*,\s*(.+?)\s*,?\s+and\s+(.+?)\s+links?[?.!]*$", + re.IGNORECASE, + ) + + def __init__( + self, + *args: Any, + discourse_journal_path: Path | None = None, + k102_ablations: Sequence[str] = (), + **kwargs: Any, + ) -> None: + substrate_path = Path(args[5] if len(args) >= 6 else kwargs["substrate_path"]) + self._k102_ablations = frozenset(clean(value).casefold() for value in k102_ablations) + kwargs.setdefault("default_system_prompt", K102_SYSTEM_PROMPT) + super().__init__(*args, **kwargs) + self.graph_transducer = K102FinitePredicateTransducer(self, self.construction_head) + self.inverse = QuadraticAnswerInverse(K102SituatedInverseGrammar(self.dictionary)) + source_sha = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + self.discourse_journal = IndexedTypedDiscourseJournal( + discourse_journal_path or substrate_path.with_suffix(".k102.discourse.sqlite3"), + source_sha256=source_sha, + ) + self._k102_bound_constitutions: dict[str, str] = {} + self._k102_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k100_lineage_sha256, + "opened_failure_source": "stage5k102_long_horizon_development_probe_v1", + "repairs": [ + "typed_situated_identity_constitution", + "indexed_hash_chained_discourse_workspace", + "active_comparison_and_decision_frames", + "causal_retraction_replacement_and_propagation", + "topic_stack_suspend_and_restore", + "rhetorical_artifact_construction", + "mediated_cross_domain_synthesis", + "tool_observation_continuity_and_comparison", + "finite_surface_tense_and_productive_nominal_number", + "dynamic_query_supplied_synthesis_path_labels", + "active_topic_exact_posting_evidence_grounding", + "indefinite_article_rhetorical_construction_equivalence", + ], + }) + + def _tree_record(self, session_id: str, turn_id: str) -> dict[str, Any]: + record = super()._tree_record(session_id, turn_id) + current = _K102_TURN.get() + profile = compile_identity_profile(self.default_system_prompt) + constitution = current.constitution if current else compile_situated_constitution(profile) + state_sha = current.state.state_sha256 if current else ZERO_SHA256 + spine = record.setdefault("dual_rail_contract", {}).setdefault("stabilizing_spine", {}) + spine["stage5k102:spine:situated_identity_discourse"] = { + "constitution_sha256": constitution.constitution_sha256, + "discourse_state_sha256": state_sha, + "constitutional_roles_bound": 11, + "discourse_payload_unbound": False, + "mutable_during_tree_transit": False, + "unbind_permission": "malkuth_only", + } + return record + + def _bind_memory( + self, + session_id: str, + turn_id: str, + roles: Mapping[str, str], + *, + bank: str = "episodic", + source: str = "native_mixed_agent", + ) -> str: + # K51 reconstructs its ephemeral workspace by executing committed + # typed operations again. Their HRR side effects are already durable; + # attempting to append them a second time correctly collides with the + # substrate event key. Return an opaque deterministic replay handle + # while reconstructing, then resume ordinary binding for the new turn. + if getattr(self, "_k51_replaying", False): + return "mem_replay_" + digest({ + "session_id": session_id, + "turn_id": turn_id, + "roles": dict(roles), + "bank": bank, + "source": source, + })[:48] + return super()._bind_memory( + session_id, turn_id, roles, bank=bank, source=source, + ) + + @staticmethod + def _push_topic(state: TypedDiscourseState, topic: str) -> None: + topic = _trim(topic) + if not topic: + return + if state.active_topic and state.active_topic.casefold() != topic.casefold(): + if not state.topic_stack or state.topic_stack[-1].casefold() != state.active_topic.casefold(): + state.topic_stack.append(state.active_topic) + state.active_topic = topic + + @staticmethod + def _proof_arguments(output: Mapping[str, Any]) -> dict[str, Any]: + return { + str(key): value + for key, value in (output.get("computation_proof", {}) or {}).get("arguments", ()) + } + + @staticmethod + def _proof_clauses(output: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: + return tuple((output.get("computation_proof", {}) or {}).get("clauses", ())) + + @staticmethod + def _atom_identity(atom: Mapping[str, Any]) -> str: + return _sha({ + "subject": clean(atom.get("subject", "")).casefold(), + "predicate": clean(atom.get("predicate", "")).casefold(), + "object": clean(atom.get("object", atom.get("object_value", ""))).casefold(), + "world": clean(atom.get("world", "actual")).casefold(), + "polarity": clean(atom.get("polarity", "positive")).casefold(), + }) + + @staticmethod + def _atom_from_mapping(value: Mapping[str, Any]) -> GraphAtom: + return GraphAtom( + clean(value.get("subject", "entity")), + clean(value.get("predicate", "relate")), + clean(value.get("object", value.get("object_value", "event"))) or "event", + world=clean(value.get("world", "actual")), + polarity=clean(value.get("polarity", "positive")), + qualifiers=tuple((clean(k), clean(v)) for k, v in value.get("qualifiers", ())), + ) + + @staticmethod + def _edge_chain(state: TypedDiscourseState, requested_effect: str | None = None) -> list[dict[str, Any]]: + edges = [value for value in state.causal_edges if value.get("active", True)] + if not edges: + return [] + outgoing = {PersistentTypedDiscourseAgent._atom_identity(value["cause"]) for value in edges} + sinks = [value["effect"] for value in edges if PersistentTypedDiscourseAgent._atom_identity(value["effect"]) not in outgoing] + target = None + if requested_effect: + request_terms = set(tokens(requested_effect)) + ranked = sorted( + ( + len(request_terms & set(tokens(clean(value["effect"].get("subject", "")) + " " + clean(value["effect"].get("predicate", ""))))), + index, + value["effect"], + ) + for index, value in enumerate(edges) + ) + if ranked and ranked[-1][0] > 0: + target = ranked[-1][2] + target = target or (sinks[-1] if sinks else edges[-1]["effect"]) + chain = [target] + visited = {PersistentTypedDiscourseAgent._atom_identity(target)} + while True: + target_id = PersistentTypedDiscourseAgent._atom_identity(chain[-1]) + edge = next((value for value in reversed(edges) if PersistentTypedDiscourseAgent._atom_identity(value["effect"]) == target_id), None) + if edge is None: + break + cause_id = PersistentTypedDiscourseAgent._atom_identity(edge["cause"]) + if cause_id in visited: + break + chain.append(edge["cause"]) + visited.add(cause_id) + return list(reversed(chain)) + + def _constitutional_identity_result(self, constitution: SituatedIdentityConstitution) -> HandlerResult: + return HandlerResult( + # Preserve the established public operation contract while the + # proof itself carries K102's richer situated constitution. + "identity_purpose", "answered", + (("identity", constitution.self_name),), + (("constitution_sha256", constitution.constitution_sha256), ("constitutional_roles", 11)), + ( + SemanticClause("I", "be", complement=constitution.self_name), + SemanticClause("My operative identity", "be", complement="a persistent typed relational constitution bound to the stabilizing spine"), + SemanticClause("My purpose", "be", complement=constitution.purpose), + SemanticClause("I", "treat", object_value="each utterance as a change to a shared evolving situation"), + SemanticClause("I", "preserve", object_value="topics, referents, commitments, corrections, goals, evidence, and tool observations as typed discourse state"), + SemanticClause("The user", "retain", object_value="authority over goals, meanings, corrections, and consequential choices"), + SemanticClause("My open-domain competence", "remain", complement="experimental until it survives fresh long-horizon evaluations"), + ), + (("source", "k102_native_situated_identity_constitution"),), + ( + "bind_constitution_to_protected_root", + "distinguish_persistent_self_from_mutable_situation", + "bind_partner_and_authority_relation", + "compose_native_self_description", + ), + ) + + def _topic_result(self, topic: str) -> HandlerResult: + return HandlerResult( + "conversational_topic_invitation", "answered", (("topic", topic),), (("accepted", True),), + ( + SemanticClause("We", "can", object_value=f"discuss {topic}"), + SemanticClause("I", "treat", object_value="it as the active topic and preserve it across follow-up questions and temporary shifts", modal="will"), + SemanticClause("You", "can", object_value="begin with the definition, mechanism, evidence, consequence, or uncertainty that matters most"), + ), + (("source", "typed_topic_stack"),), + ("recognize_topic_invitation", "push_prior_topic", "bind_active_topic", "offer_relational_entry_points"), + ) + + def _comparison_followup(self, state: TypedDiscourseState, text: str) -> HandlerResult | None: + frame = state.comparison + if not frame: + return None + lower = clean(text).casefold() + left, right = str(frame["left"]), str(frame["right"]) + left_definition = str(frame.get("left_definition", "")) + right_definition = str(frame.get("right_definition", "")) + # K73's productive normalization can map number words to digits before + # handler dispatch. Both surfaces denote the same construction. + if re.search(r"\b(?:one|1|single)\s+sentence\b|\bin\s+(?:one|1)\s+sentence\b", lower): + return HandlerResult( + "summarize_active_comparison", "answered", (("left", left), ("right", right)), + (("sentence_count", 1),), + (SemanticClause( + f"{left} and {right}", "differ", + qualifiers=(("because", f"{left} is {left_definition}, whereas {right} is {right_definition}"),), + ),), + (("source", "active_definition_grounded_comparison"),), + ("restore_active_comparison", "retain_selected_lexical_senses", "compress_shared_contrast"), + ) + match = re.match(r"^which\s+(?:one|1)\s+(.+?)[?.!]*$", clean(text), re.IGNORECASE) + if match is None: + return None + requested = _trim(match.group(1)) + requested_terms = set(tokens(requested)) + definitions = {left: left_definition, right: right_definition} + scored = [] + for label, definition in definitions.items(): + definition_terms = set(tokens(definition)) + overlap = len(requested_terms & definition_terms) + excluded = bool( + overlap + and re.search(r"\b(?:rather\s+than|not|without|instead\s+of)\b", definition, re.IGNORECASE) + ) + scored.append((overlap, excluded, label)) + excluded = [value for value in scored if value[0] and value[1]] + if len(excluded) == 1: + selected = right if excluded[0][2] == left else left + warrant = f"{excluded[0][2]}'s grounded definition explicitly contrasts itself with {requested}" + else: + ranked = sorted(scored, key=lambda value: (value[0], not value[1], value[2])) + if not ranked or ranked[-1][0] == 0: + return HandlerResult( + "clarify_active_comparison_dimension", "clarification", + (("left", left), ("right", right)), (("dimension", requested),), + (SemanticClause("I", "need", object_value=f"the intended relation between {requested} and the active comparison"),), + (("source", "active_comparison_has_no_grounded_dimension_match"),), + ("restore_active_comparison", "test_requested_dimension", "preserve_unresolved_role"), + ) + selected = ranked[-1][2] + warrant = f"its grounded definition has the strongest positive relation to {requested}" + other = right if selected == left else left + return HandlerResult( + "query_active_comparison", "answered", (("left", left), ("right", right), ("dimension", requested)), + (("selected", selected), ("other", other)), + ( + SemanticClause(f"{selected} rather than {other}", "be", complement=f"the side of the active comparison that {requested}"), + SemanticClause("The warrant", "be", complement=warrant), + ), + (("source", "active_definition_grounded_comparison"),), + ("restore_active_comparison", "bind_requested_dimension", "evaluate_definition_polarity", "release_selected_role"), + ) + + def _decision_followup(self, state: TypedDiscourseState, text: str) -> HandlerResult | None: + frame = state.decision + if not frame: + return None + value, lower = clean(text), clean(text).casefold() + concern_match = self._CONCERN.match(value) + if concern_match: + concern = _trim(concern_match.group(1)) + return HandlerResult( + "bind_decision_constraint", "answered", (("decision", frame["question"]),), + (("constraint", concern),), + ( + SemanticClause(concern, "be", complement="the leading concern in the active decision"), + SemanticClause("The decision analysis", "give", object_value="that concern priority without treating it as the only relevant consideration", modal="will"), + ), + (("source", "active_typed_decision_frame"),), + ("restore_decision_frame", "bind_user_stated_constraint", "preserve_user_authority"), + ) + concerns = [str(value) for value in frame.get("concerns", ())] + concern = concerns[-1] if concerns else "the largest uncertainty" + if re.search(r"\bwhat\s+should\s+i\s+examine\s+first\b|\bwhere\s+should\s+i\s+start\b", lower): + return HandlerResult( + "prioritize_decision_inquiry", "answered", (("decision", frame["question"]),), + (("leading_constraint", concern),), + ( + SemanticClause( + "You", "examine", object_value=concern, + qualifiers=(("first", "by making it concrete and measurable"),), modal="should", + ), + SemanticClause("That examination", "separate", object_value="downside exposure, available buffer, time horizon, and recovery options", modal="should"), + SemanticClause("A reversible test", "show", object_value="how much of the uncertainty can be reduced before the full decision is made", modal="can"), + ), + (("source", "native_decision_relation_schema"),), + ("restore_decision_frame", "rank_user_constraint", "decompose_exposure_and_reversibility", "preserve_choice_authority"), + ) + if re.search(r"\bsummarize\b.*\bdecision\b|\bdecision\s+structure\b", lower): + concerns_text = ", ".join(concerns) if concerns else "no dominant constraint has yet been named" + return HandlerResult( + "summarize_active_decision", "answered", (("decision", frame["question"]),), + (("concerns", concerns),), + ( + SemanticClause("The active decision", "concern", object_value=str(frame["question"])), + SemanticClause("The comparison", "include", object_value="the desired outcome, alternatives, benefits, costs, timing, uncertainty, and reversibility"), + SemanticClause("The leading constraint", "be", complement=concerns_text), + SemanticClause("The next inquiry", "be", complement=f"to measure {concern} and identify a reversible way to reduce it"), + ), + (("source", "active_typed_decision_frame"),), + ("restore_decision_frame", "collect_bound_constraints", "compose_deliberative_summary"), + ) + return None + + def _paper_result(self, state: TypedDiscourseState, text: str) -> HandlerResult | None: + start = self._PAPER_START.match(clean(text)) + if start and "rhetorical_article_equivalence" in self._k102_ablations: + surface = WORD.findall(clean(text).casefold()) + if "an" in surface: + start = None + if start: + claim = _trim(start.group(1)) + return HandlerResult( + "initialize_rhetorical_artifact", "answered", (("genre", "paper"), ("claim", claim)), + (("artifact_state", "initialized"),), + ( + SemanticClause("The working claim", "be", complement=f"that {claim}"), + SemanticClause("The paper", "define", object_value="its terms, construct its positive case, test an opposing explanation, and state the conditions under which the claim holds", modal="will"), + ), + (("source", "native_rhetorical_construction_algebra"),), + ("recognize_rhetorical_goal", "bind_claim", "initialize_revisable_artifact"), + ) + frame = state.paper + if not frame: + return None + lower = clean(text).casefold() + claim = str(frame["claim"]) + if all(word in lower for word in ("thesis", "section", "objection", "conclusion")): + objection = ( + "modularity can merely move complexity into interfaces, where cross-component interactions " + "and distributed responsibility become difficult to inspect" + ) + conclusion = ( + f"{claim}, provided that interfaces, provenance, composition rules, and system-level behavior remain testable" + ) + return HandlerResult( + "construct_rhetorical_artifact", "answered", (("genre", "paper"), ("claim", claim)), + (("sections", 3), ("objection", objection), ("conclusion", conclusion)), + ( + SemanticClause("The thesis", "be", complement=f"that {claim}"), + SemanticClause("Section one", "define", object_value="the units of comparison, the meaning of auditability, and the scope of the claim"), + SemanticClause("Section two", "explain", object_value="how decomposition, explicit interfaces, local tests, and provenance can expose the path from evidence to output"), + SemanticClause("Section three", "compare", object_value="those benefits with monolithic integration, end-to-end optimization, and system-level failure modes"), + SemanticClause("The strongest objection", "be", complement=objection), + SemanticClause("The conclusion", "be", complement=conclusion), + ), + (("source", "native_rhetorical_construction_algebra"),), + ("restore_artifact", "instantiate_thesis", "order_supporting_sections", "construct_strong_objection", "bind_conditional_conclusion"), + ) + if re.search(r"\bstrengthen\b.*\bobjection\b", lower): + objection = ( + "a modular system may be less auditable in practice when emergent failures arise only from interactions among individually transparent components, while interface proliferation diffuses causal and institutional responsibility" + ) + return HandlerResult( + "revise_rhetorical_objection", "answered", (("claim", claim),), + (("thesis_preserved", True), ("objection", objection)), + ( + SemanticClause("The strengthened objection", "be", complement=objection), + SemanticClause("The thesis", "survive", object_value="only if the audit covers compositions and interfaces as well as individual modules"), + ), + (("source", "active_rhetorical_artifact"),), + ("restore_artifact", "preserve_thesis", "increase_opposing_explanatory_scope", "revise_conditional_warrant"), + ) + return None + + def _synthesis_result(self, state: TypedDiscourseState, text: str) -> HandlerResult | None: + start = self._SYNTHESIS_START.match(clean(text)) + if start: + source, left, right = (_trim(start.group(index)) for index in (1, 2, 3)) + return HandlerResult( + "construct_mediated_cross_domain_synthesis", "answered", + (("source", source), ("endpoint_left", left), ("endpoint_right", right)), + (("paths", 3), ("epistemic_status", "conditional_hypothesis")), + ( + SemanticClause(source, "alter", object_value=f"physical resources or processes relevant to {left}", modal="may"), + SemanticClause("The resulting constraint", "affect", object_value=f"{left} through availability, demand, substitution, and adaptation", modal="may"), + SemanticClause(f"Changes in {left}", "affect", object_value=f"{right} through distribution, institutional response, and public trust", modal="may"), + SemanticClause("The last link", "be", complement="the least direct because it contains the most mediators and competing causes"), + SemanticClause("This chain", "be", complement="a conditional mechanism to test rather than an established prediction"), + ), + (("source", "native_mediated_causal_schema"),), + ("bind_source_and_two_endpoints", "separate_physical_economic_and_institutional_paths", "rank_mediation_depth", "preserve_hypothetical_status"), + ) + frame = state.synthesis + if not frame: + return None + lower = clean(text).casefold() + source, left, right = (str(frame[key]) for key in ("source", "left", "right")) + requested_paths = None if "dynamic_synthesis_paths" in self._k102_ablations else self._SYNTHESIS_PATHS.match(clean(text)) + if requested_paths or re.search(r"\bseparate\b.*\blinks?\b", lower): + paths = tuple( + re.sub(r"^the\s+", "", _trim(requested_paths.group(index)), flags=re.IGNORECASE) + for index in range(1, 4) + ) if requested_paths else ("physical", "economic", "institutional") + return HandlerResult( + "decompose_active_cross_domain_synthesis", "answered", + (("source", source), ("endpoint_left", left), ("endpoint_right", right)), + (("paths", list(paths)),), + ( + SemanticClause(f"The {paths[0]} link", "run", qualifiers=(("from", source), ("through", f"resource and process constraints relevant to {left}"))), + SemanticClause(f"The {paths[1]} link", "run", qualifiers=(("from", "those constraints"), ("through", f"availability, demand, substitution, and adaptation to {left}"))), + SemanticClause(f"The {paths[2]} link", "run", qualifiers=(("from", f"the distributional consequences of {left}"), ("through", f"institutions and public response to {right}"))), + SemanticClause("All three links", "remain", complement="conditional until their mediators are supported by evidence for the case at hand"), + ), + (("source", "active_mediated_causal_schema"),), + ("restore_synthesis", "bind_query_supplied_path_labels", "unbind_named_paths_at_terminal", "preserve_mediator_order"), + ) + if re.search(r"\b(?:least|most)\s+(?:certain|uncertain)\b", lower): + paths = tuple(str(value) for value in frame.get("paths", ())) + least_certain = paths[-1] if paths else "institutional" + return HandlerResult( + "rank_active_synthesis_uncertainty", "answered", + (("source", source), ("endpoint", right)), (("least_certain_path", least_certain),), + ( + SemanticClause(f"The {least_certain} link", "be", complement="the least certain of the three current paths"), + SemanticClause("It", "contain", object_value="the greatest number of mediators, institutional contingencies, feedback effects, and competing causes"), + SemanticClause("That ranking", "compare", object_value="structural uncertainty within this hypothesis; it does not establish the other links as facts"), + ), + (("source", "active_mediated_causal_schema"),), + ("restore_synthesis", "count_mediating_relations", "rank_structural_uncertainty", "retain_epistemic_scope"), + ) + return None + + def _active_topic_evidence_result( + self, state: TypedDiscourseState, text: str, + ) -> HandlerResult | None: + """Ground evidence follow-ups in the restored topic, not request words.""" + topic = clean(state.active_topic or "") + if "active_topic_evidence" in self._k102_ablations or not topic or not re.search( + r"\b(?:evidence|support|supports|substantiate|substantiates|source|sources)\b", + clean(text), re.IGNORECASE, + ): + return None + rows = tuple( + row for row in self._topic_source_rows(topic, limit=96) + if clean(row.get("fact", "")) and not clean(row.get("fact", "")).rstrip().endswith("?") + ) + if not rows: + return HandlerResult( + "clarify_active_topic_evidence", "clarification", (("topic", topic),), + (("missing_role", "an authenticated source proposition about the active topic"),), + (SemanticClause("I", "need", object_value=f"an authenticated source proposition about {topic} before treating the requested support as evidence"),), + (("source", "active_topic_exact_source_lookup_empty"),), + ("restore_active_topic", "require_exact_topic_source", "preserve_epistemic_boundary"), + ) + selected = rows[:2] + clauses = tuple( + SemanticClause( + "The cited source" if len(selected) == 1 else f"Cited source {index}", + "state", object_value=clean(row["fact"]).rstrip(".?!"), + ) + for index, row in enumerate(selected, start=1) + ) + evidence = tuple( + (str(row.get("source_document_sha256", row.get("source_document", ""))), str(row.get("text_sha256", ""))) + for row in selected + ) + return HandlerResult( + "ground_active_topic_evidence", "answered", (("topic", topic),), + (("source_rows", len(selected)),), clauses, evidence, + ( + "restore_active_topic", "resolve_deictic_topic_reference", + "require_exact_topic_posting", "rank_declarative_source_propositions", + "release_authenticated_evidence_at_malkuth", + ), + ) + + def _revise_causal_result(self, session_id: str, state: TypedDiscourseState, text: str) -> HandlerResult | None: + match = self._CAUSAL_CORRECTION.match(clean(text)) + if match is None: + return None + subject, old_surface, new_surface, cause_surface = (_trim(match.group(index)) for index in range(1, 5)) + old_atom_candidates = [] + discourse = self._graph_discourse(session_id) + old_terms = set(tokens(old_surface)) + for fact in reversed(discourse.facts): + if not fact.active or fact.atom.subject.casefold() != subject.casefold(): + continue + if set(tokens(fact.atom.predicate + " " + fact.atom.object_value)) & old_terms: + old_atom_candidates.append(fact) + old_fact = old_atom_candidates[0] if old_atom_candidates else None + downstream = [] + if old_fact is not None: + downstream_ids = [right for left, right in discourse.causal_edges if left == old_fact.state_id] + downstream = [value for value in discourse.facts if value.active and value.state_id in downstream_ids] + old_fact.active = False + session = self._session(session_id) + if session.workspace.is_visible(old_fact.state_id): + session.workspace.retract(old_fact.state_id, reason="k102_causal_correction") + discourse.causal_edges = [ + edge for edge in discourse.causal_edges if old_fact.state_id not in edge + ] + new_effect = self.graph_transducer._simple_clause(f"{subject} {new_surface}") + cause = self.graph_transducer._simple_clause(cause_surface) + if new_effect is None or cause is None: + return HandlerResult( + "clarify_causal_revision", "clarification", (("subject", subject),), + (("unresolved", "replacement cause or effect"),), + (SemanticClause("I", "need", object_value="the replacement cause and the event it produced"),), + (("source", "typed_causal_revision_parser"),), + ("retain_negated_commitment", "identify_unresolved_replacement_role"), + ) + self._assert_causal(session_id, (cause, new_effect)) + for fact in downstream: + self._assert_causal(session_id, (new_effect, fact.atom)) + retracted_identity = self._atom_identity(old_fact.atom.as_dict()) if old_fact else None + revised_edges = [ + value for value in state.causal_edges + if retracted_identity is None + or ( + self._atom_identity(value["cause"]) != retracted_identity + and self._atom_identity(value["effect"]) != retracted_identity + ) + ] + revised_edges.append({"cause": cause.as_dict(), "effect": new_effect.as_dict(), "active": True}) + revised_edges.extend( + {"cause": new_effect.as_dict(), "effect": fact.atom.as_dict(), "active": True} + for fact in downstream + ) + current = _K102_TURN.get() + if current is not None: + current.audit.update({ + "recognized": True, + "discourse_operation": "causal_revision", + "retracted_atom": old_fact.atom.as_dict() if old_fact else None, + "replacement_cause": cause.as_dict(), + "replacement_effect": new_effect.as_dict(), + "revised_causal_edges": revised_edges, + }) + chain = [cause, new_effect, *(value.atom for value in downstream)] + return HandlerResult( + "revise_causal_discourse", "answered", (("subject", subject),), + (("retracted", old_fact.atom.as_dict() if old_fact else None), ("replacement_edges", max(1, len(chain) - 1))), + tuple(self._past_atom_clause(value) for value in chain), + (("source", "conversation_local_causal_revision"),), + ("resolve_corrected_subject", "retract_superseded_cause", "preserve_unaffected_downstream_effect", "bind_replacement_path", "propagate_revision"), + ) + + def _causal_query_result(self, state: TypedDiscourseState, text: str) -> HandlerResult | None: + if not state.causal_edges: + return None + lower = clean(text).casefold() + if not ( + re.match(r"^why\b", lower) + or re.search(r"\b(?:whole\s+sequence|causal\s+order|what\s+(?:now\s+)?explains?)\b", lower) + ): + return None + chain = self._edge_chain(state, text) + if len(chain) < 2: + return None + atoms = tuple(self._atom_from_mapping(value) for value in chain) + return HandlerResult( + "query_persistent_causal_discourse", "answered", (("path_length", len(atoms)),), + (("atoms", [value.as_dict() for value in atoms]),), + tuple(self._past_atom_clause(value) for value in atoms), + (("source", "persistent_typed_causal_discourse"),), + ("restore_causal_edges", "select_requested_downstream_effect", "traverse_active_reverse_edges", "release_revised_chain"), + ) + + def _tool_continuity_result(self, state: TypedDiscourseState, text: str) -> HandlerResult | None: + prior = state.last_tool + lower = clean(text).casefold() + if not prior or not re.search(r"\bcompare\b", lower) or not re.search(r"\b(?:strongest|largest|greatest|maximum)\b", lower): + return None + backend = self.tool_backend + expert = getattr(backend, "experts", {}).get(prior.get("expert_id")) if backend else None + field_name = str(prior.get("field") or "") + if expert is None or not field_name: + return None + specification = next((value for value in expert.schema.fields if value.name == field_name), None) + if specification is None or specification.kind not in {"integer", "number"}: + return None + previous_row = next(( + value for value in expert.rows + if str(value.get("event_id") or value.get("country_name") or "") == str(prior.get("row_key")) + ), None) + if previous_row is None: + return None + strongest = max(expert.rows, key=lambda value: float(value[field_name])) + previous_value = float(previous_row[field_name]) + strongest_value = float(strongest[field_name]) + difference = abs(strongest_value - previous_value) + display = lambda value: str(int(value)) if float(value).is_integer() else format(float(value), ".12g") + strongest_key = str(strongest.get("event_id") or strongest.get("country_name") or "registered row") + previous_key = str(prior["row_key"]) + values = [display(previous_value), display(strongest_value), display(difference)] + audit = { + "schema_version": SCHEMA, + "operation": "compare", + "expert_id": expert.schema.expert_id, + "row_key": previous_key, + "comparison_row_key": strongest_key, + "field": field_name, + "values": values, + "read_only": True, + "typed_arguments_bound": True, + "goal_postconditions_satisfied": True, + "source_row_restored_from_discourse": True, + "traditional_language_model": False, + } + return HandlerResult( + "bounded_agentic_tool", "answered", + (("operation", "compare"), ("expert_id", expert.schema.expert_id), ("field", field_name)), + (("values", values), ("difference", values[2])), + ( + SemanticClause(f"The {field_name} for {previous_key}", "be", complement=values[0]), + SemanticClause(f"The maximum {field_name}", "be", complement=values[1], qualifiers=(("at", strongest_key),)), + SemanticClause(f"The absolute {field_name} difference", "be", complement=values[2]), + ), + (("expert_id", expert.schema.expert_id),), + ("restore_typed_tool_observation", "select_registered_numeric_field", "execute_read_only_extremum", "compute_typed_difference", "verify_goal_postconditions"), + tool_audit=audit, + ) + + def _discourse_result(self, session_id: str, text: str) -> HandlerResult | None: + current = _K102_TURN.get() + if current is None: + return None + state = current.state + if self._identity_request(text): + current.audit.update({"recognized": True, "discourse_operation": "situated_identity"}) + return self._constitutional_identity_result(current.constitution) + correction = self._revise_causal_result(session_id, state, text) + if correction is not None: + return correction + causal_query = self._causal_query_result(state, text) + if causal_query is not None: + current.audit.update({"recognized": True, "discourse_operation": "persistent_causal_query"}) + return causal_query + tool = self._tool_continuity_result(state, text) + if tool is not None: + current.audit.update({"recognized": True, "discourse_operation": "tool_observation_comparison"}) + return tool + shift = self._SHIFT_COMPARE.match(clean(text)) + if shift: + left, right = (_trim(shift.group(index)) for index in (1, 2)) + result = self._situated_result(f"Compare {left} and {right}.") + if result is not None: + current.audit.update({ + "recognized": True, "discourse_operation": "suspend_topic_and_compare", + "left": left, "right": right, + }) + return HandlerResult( + "discourse_topic_shift_comparison", result.status, result.arguments, result.result, + result.clauses, result.evidence, + ("suspend_active_topic", *result.computation_trace, "bind_temporary_comparison_topic"), + tool_audit=result.tool_audit, memory_audit=result.memory_audit, + ) + returning = self._RETURN_TOPIC.match(clean(text)) + if returning: + target = state.topic_stack[-1] if state.topic_stack else state.active_topic + if target: + current.audit.update({"recognized": True, "discourse_operation": "restore_topic", "topic": target}) + return HandlerResult( + "restore_prior_topic", "answered", (("topic", target),), (("restored", True),), + ( + SemanticClause("Our earlier topic", "be", complement=target), + SemanticClause("We", "examine", object_value=f"{target} through its active questions and relations", tense="past"), + ), + (("source", "typed_topic_stack"),), + ("read_topic_stack", "pop_suspended_topic", "restore_discourse_focus"), + ) + topic = self._TOPIC_INVITATION.match(clean(text)) + if topic: + subject = _trim(topic.group(1)) + current.audit.update({"recognized": True, "discourse_operation": "bind_topic", "topic": subject}) + return self._topic_result(subject) + paper = self._paper_result(state, text) + if paper is not None: + current.audit.update({"recognized": True, "discourse_operation": paper.operation}) + return paper + synthesis = self._synthesis_result(state, text) + if synthesis is not None: + current.audit.update({"recognized": True, "discourse_operation": synthesis.operation}) + return synthesis + topic_evidence = self._active_topic_evidence_result(state, text) + if topic_evidence is not None: + current.audit.update({ + "recognized": True, + "discourse_operation": topic_evidence.operation, + "topic": state.active_topic, + }) + return topic_evidence + comparison = self._comparison_followup(state, text) + if comparison is not None: + current.audit.update({"recognized": True, "discourse_operation": comparison.operation}) + return comparison + decision = self._decision_followup(state, text) + if decision is not None: + current.audit.update({"recognized": True, "discourse_operation": decision.operation}) + return decision + return None + + @staticmethod + def _resolve_topic_anaphor(state: TypedDiscourseState, text: str) -> tuple[str, str | None]: + topic = clean(state.active_topic or "") + value = clean(text) + if not topic: + return value, None + patterns = ( + r"\b(?:it|its)\b", + r"\b(?:this|that)\s+(?:topic|subject|process|system|idea|claim)\b", + ) + for pattern in patterns: + if re.search(pattern, value, re.IGNORECASE): + return re.sub(pattern, topic, value, count=1, flags=re.IGNORECASE), topic + return value, None + + @staticmethod + def _with_topic_resolution( + result: HandlerResult, + *, + topic: str, + original_text: str, + ) -> HandlerResult: + return HandlerResult( + result.operation, + result.status, + (*result.arguments, ("resolved_topic_anaphor", topic)), + result.result, + result.clauses, + (*result.evidence, ("topic_resolution", digest({"topic": topic, "surface": clean(original_text)}))), + ("restore_active_topic", "resolve_deictic_topic_reference", *result.computation_trace), + tool_audit=result.tool_audit, + memory_audit=result.memory_audit, + ) + + def _repair_graph_assertion_surface( + self, result: HandlerResult | None, surface: str, + ) -> HandlerResult | None: + if ( + result is None + or result.operation != "assert_graph_fact" + or "finite_number_recovery" in self._k102_ablations + ): + return result + atom = self.graph_transducer._simple_clause(surface) + finite = self.graph_transducer.finite_surface(surface, atom) if atom is not None else None + if atom is None or finite is None: + return result + lemma, _listed = self.dictionary.lemma(finite) + exceptions = self.dictionary.reverse_exceptions.get(lemma, ()) + is_past = bool( + finite in exceptions + or finite.endswith(("ed", "ied", "en")) + or re.search(r"\b(?:did|was|were|had)\b", clean(surface), re.IGNORECASE) + ) + clause = self._atom_clause(atom) + if is_past and atom.predicate not in {"is_a", "property", "be"}: + clause = replace(clause, tense="past") + return HandlerResult( + result.operation, result.status, result.arguments, result.result, + (clause,), result.evidence, + (*result.computation_trace, "recover_finite_surface_tense_and_nominal_number"), + tool_audit=result.tool_audit, memory_audit=result.memory_audit, + ) + + def _handle_evidence(self, text: str, context: Mapping[str, Any]) -> HandlerResult | None: + current = _K102_TURN.get() + session_id = current.session_id if current else str(context.get("k102_session_id", "")) + result = self._discourse_result(session_id, text) if session_id else None + if result is not None: + return result + resolved, topic = self._resolve_topic_anaphor(current.state, text) if current else (text, None) + inherited = self._repair_graph_assertion_surface( + super()._handle_evidence(resolved, context), resolved, + ) + if inherited is not None and topic is not None: + current.audit.update({ + "recognized": True, + "discourse_operation": "resolve_topic_anaphor", + "topic": topic, + "resolved_surface_sha256": hashlib.sha256(resolved.encode("utf-8")).hexdigest(), + }) + return self._with_topic_resolution(inherited, topic=topic, original_text=text) + return inherited + + def _handle_event(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + result = self._discourse_result(session_id, text) + if result is not None: + return result + current = _K102_TURN.get() + resolved, topic = self._resolve_topic_anaphor(current.state, text) if current else (text, None) + inherited = self._repair_graph_assertion_surface( + super()._handle_event(session_id, resolved, turn_id), resolved, + ) + if inherited is not None and topic is not None: + current.audit.update({ + "recognized": True, + "discourse_operation": "resolve_topic_anaphor", + "topic": topic, + "resolved_surface_sha256": hashlib.sha256(resolved.encode("utf-8")).hexdigest(), + }) + return self._with_topic_resolution(inherited, topic=topic, original_text=text) + return inherited + + def _handle_graph(self, session_id: str, graph: Any) -> HandlerResult | None: + result = super()._handle_graph(session_id, graph) + if result is None or result.operation != "assert_graph_fact": + return result + current = _K102_TURN.get() + surface = clean(current.input_surface) if current else "" + return self._repair_graph_assertion_surface(result, surface) if surface else result + + def _event_capability_family(self, text: str) -> tuple[str, ...]: + lower = clean(text).casefold() + if ( + self._TOPIC_INVITATION.match(clean(text)) + or self._RETURN_TOPIC.match(clean(text)) + or self._SHIFT_COMPARE.match(clean(text)) + or self._PAPER_START.match(clean(text)) + or self._SYNTHESIS_START.match(clean(text)) + or self._CAUSAL_CORRECTION.match(clean(text)) + or re.search(r"\b(?:thesis|section|objection|conclusion|concern|(?:one|1)\s+sentence|earlier\s+topic)\b", lower) + ): + return ("linguistic", "synthesis", "epistemic", "contrast", "causal", "mechanism", "tool_router", "dataset", "provenance") + return super()._event_capability_family(text) + + def _record_parent_state( + self, + state: TypedDiscourseState, + text: str, + output: Mapping[str, Any], + audit: Mapping[str, Any], + ) -> None: + operation = str(output.get("operation", "")) + arguments = self._proof_arguments(output) + clauses = self._proof_clauses(output) + if operation in {"relational_lexical_comparison", "discourse_topic_shift_comparison"}: + left, right = clean(arguments.get("left", audit.get("left", ""))), clean(arguments.get("right", audit.get("right", ""))) + complements = [clean(value.get("complement", "")) for value in clauses if value.get("complement")] + if left and right: + state.comparison = { + "left": left, "right": right, + "left_definition": complements[0] if complements else "the first grounded concept", + "right_definition": complements[1] if len(complements) > 1 else "the second grounded concept", + } + self._push_topic(state, f"comparison of {left} and {right}") + elif operation == "conversational_decision_frame": + question = clean(arguments.get("decision", text)) + state.decision = {"question": question, "concerns": []} + self._push_topic(state, question) + elif operation == "bind_decision_constraint" and state.decision: + result = {str(key): value for key, value in (output.get("computation_proof", {}) or {}).get("result", ())} + concern = clean(result.get("constraint", "")) + if concern and concern not in state.decision.setdefault("concerns", []): + state.decision["concerns"].append(concern) + elif operation == "conversational_topic_invitation": + topic = clean(arguments.get("topic", audit.get("topic", ""))) + if topic: + self._push_topic(state, topic) + elif operation == "restore_prior_topic": + target = clean(audit.get("topic", "")) + if target: + if state.active_topic and state.active_topic.casefold().startswith("comparison of "): + state.comparison = None + if state.topic_stack and state.topic_stack[-1].casefold() == target.casefold(): + state.topic_stack.pop() + state.active_topic = target + elif operation == "initialize_rhetorical_artifact": + claim = clean(arguments.get("claim", "")) + state.paper = {"claim": claim, "revision": 0} + if claim: + self._push_topic(state, claim) + elif operation in {"construct_rhetorical_artifact", "revise_rhetorical_objection"} and state.paper: + state.paper["revision"] = int(state.paper.get("revision", 0)) + 1 + result = {str(key): value for key, value in (output.get("computation_proof", {}) or {}).get("result", ())} + for key in ("sections", "objection", "conclusion"): + if key in result: + state.paper[key] = result[key] + elif operation == "construct_mediated_cross_domain_synthesis": + state.synthesis = { + "source": clean(arguments.get("source", "")), + "left": clean(arguments.get("endpoint_left", "")), + "right": clean(arguments.get("endpoint_right", "")), + "epistemic_status": "conditional_hypothesis", + } + self._push_topic(state, f"effects of {state.synthesis['source']}") + elif operation == "decompose_active_cross_domain_synthesis" and state.synthesis: + result = {str(key): value for key, value in (output.get("computation_proof", {}) or {}).get("result", ())} + paths = [clean(value) for value in result.get("paths", ()) if clean(value)] + if paths: + state.synthesis["paths"] = paths + elif operation == "compose_causal_graph": + cause, effect = arguments.get("cause"), arguments.get("effect") + if isinstance(cause, Mapping) and isinstance(effect, Mapping): + edge = {"cause": dict(cause), "effect": dict(effect), "active": True} + identity = (self._atom_identity(edge["cause"]), self._atom_identity(edge["effect"])) + if not any( + (self._atom_identity(value["cause"]), self._atom_identity(value["effect"])) == identity + for value in state.causal_edges + ): + state.causal_edges.append(edge) + elif operation == "revise_causal_discourse" and audit.get("revised_causal_edges"): + state.causal_edges = [dict(value) for value in audit["revised_causal_edges"]] + tool = output.get("tool_audit") or {} + if output.get("status") == "answered" and tool.get("expert_id") and tool.get("row_key"): + state.last_tool = { + "expert_id": str(tool["expert_id"]), + "row_key": str(tool["row_key"]), + "field": None if tool.get("field") is None else str(tool["field"]), + "values": [str(value) for value in tool.get("values", ())], + "operation": str(tool.get("operation", operation)), + } + + def _bind_discourse_event( + self, + session_id: str, + output: Mapping[str, Any], + state: TypedDiscourseState, + journal: Mapping[str, Any], + ) -> tuple[str, Mapping[str, Any]]: + situation = "typed_discourse:" + "+".join(state.active_frame_names() or ("open",)) + turn_id = str(output.get("turn_id") or ("k102-" + str(journal["event_sha256"])[:20])) + memory_id = self._bind_memory( + session_id, + turn_id + "-k102-" + str(journal["sequence"]), + { + "situation": situation, + "goal": state.active_topic or "sustain situated conversation", + "transition": str(output.get("operation", "discourse_transition")), + "action": str(output.get("status", "unknown")), + "result": state.state_sha256, + "field": ",".join(state.active_frame_names()) or "open_discourse", + "status": "verified_typed_discourse_state", + "verifier": str(journal["event_sha256"]), + }, + bank="episodic", + source="k102_typed_discourse_event", + ) + return memory_id, self.substrate.cycle_once() + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + system_prompt: str | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + state, recovery = self.discourse_journal.load(session_id) + previous_state_sha = ZERO_SHA256 if not recovery["recovered"] else state.state_sha256 + profile = compile_identity_profile(system_prompt or self.default_system_prompt) + constitution = compile_situated_constitution(profile) + state.constitution_sha256 = constitution.constitution_sha256 + local_context = dict(context or {}) + local_context["k102_session_id"] = session_id + local_context["native_situated_identity_constitution"] = constitution.routing_context() + local_context["typed_discourse_state"] = { + "state_sha256": state.state_sha256, + "active_topic": state.active_topic, + "active_frames": list(state.active_frame_names()), + "turn_index": state.turn_index, + } + current = _TurnContext(session_id, state, constitution, clean(text)) + token = _K102_TURN.set(current) + try: + if self._k102_bound_constitutions.get(session_id) != constitution.constitution_sha256: + try: + self._bind_memory( + session_id, + "k102-constitution-" + constitution.constitution_sha256[:20], + { + "situation": "native_situated_identity_constitution", + "goal": constitution.purpose, + "transition": "maintain_self_across_situated_discourse_change", + "action": constitution.agency_contract, + "result": constitution.self_name, + "warrant": constitution.partner_relation, + "status": "constitutional_identity_invariant", + "verifier": constitution.constitution_sha256, + }, + bank="procedural", + source="k102_situated_identity_constitution", + ) + except ValueError as exc: + if "event_key already bound to another event" not in str(exc): + raise + self._k102_bound_constitutions[session_id] = constitution.constitution_sha256 + output = super().answer( + session_id, text, local_context, system_prompt=system_prompt, **kwargs, + ) + self._record_parent_state(state, text, output, current.audit) + state.turn_index += 1 + state.last_operation = str(output.get("operation", "")) + state.last_response_sha256 = hashlib.sha256(str(output.get("response", "")).encode("utf-8")).hexdigest() + state.constitution_sha256 = constitution.constitution_sha256 + proof_sha = str((output.get("computation_proof") or {}).get("proof_sha256", "")) + journal = self.discourse_journal.append( + session_id, + text=text, + operation=str(output.get("operation", "")), + status=str(output.get("status", "")), + previous_state_sha256=previous_state_sha, + state=state, + proof_sha256=proof_sha, + ) + memory_id, cycle = self._bind_discourse_event(session_id, output, state, journal) + spine_record = self._tree_record(session_id, str(output.get("turn_id", ""))) + spine = ( + spine_record.get("dual_rail_contract", {}) + .get("stabilizing_spine", {}) + .get("stage5k102:spine:situated_identity_discourse", {}) + ) + finally: + _K102_TURN.reset(token) + output["typed_discourse_workspace"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k102_lineage_sha256, + "development_source": "opened_k102_long_horizon_failure_probe", + "prospective_claim": False, + "state_sha256": state.state_sha256, + "turn_index": state.turn_index, + "active_topic": state.active_topic, + "topic_stack_depth": len(state.topic_stack), + "active_frames": list(state.active_frame_names()), + "journal": journal, + "recovery": recovery, + "opaque_hrr_memory_id": memory_id, + "hrr_cycle": cycle, + "bound_to_stabilizing_spine": ( + spine.get("constitution_sha256") == constitution.constitution_sha256 + and spine.get("discourse_state_sha256") == state.state_sha256 + ), + "pre_malkuth_discourse_payload_unbinds": 0, + "audit": dict(current.audit or {"recognized": False}), + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["situated_identity_constitution"] = { + "schema_version": SCHEMA, + "identity": constitution.self_name, + "profile_sha256": constitution.profile_sha256, + "constitution_sha256": constitution.constitution_sha256, + "constitutional_roles": 11, + "identity_is_typed_invariant_not_persona_overlay": True, + "prompt_is_human_readable_serialization_not_runtime_preamble": True, + "available_to_specialist_routing": True, + "persistent_procedural_binding": True, + "bound_to_stabilizing_spine": spine.get("constitution_sha256") == constitution.constitution_sha256, + "raw_prompt_prepended_to_user_utterance": False, + "situated_state_changes_without_redefining_self": True, + "user_retains_goal_and_consequential_authority": True, + } + output["runtime"]["typed_discourse_model_calls"] = 0 + output["runtime"]["typed_discourse_gradient_steps"] = 0 + output["runtime"]["typed_discourse_journal_full_scans"] = 0 + output["runtime"]["situated_identity_model_calls"] = 0 + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = [ + "IndexedTypedDiscourseJournal", + "K102_SYSTEM_PROMPT", + "PersistentTypedDiscourseAgent", + "SituatedIdentityConstitution", + "TypedDiscourseState", + "compile_situated_constitution", +] diff --git a/root/wander-train/stage5k104_relational_ingress_equivalence_agent.py b/root/wander-train/stage5k104_relational_ingress_equivalence_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..7277d142d13bf561376d56a39600fc97f90e5bfc --- /dev/null +++ b/root/wander-train/stage5k104_relational_ingress_equivalence_agent.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""K104 successor for the two ingress classes exposed prospectively by K103. + +K102 and the failed one-shot K103 evidence remain immutable. This successor +adds no reasoning cell, language model, embedding encoder, or generator. It +only maps direct rhetorical imperatives onto the existing artifact operator +and gates authoritative-science requests by exact referent-bearing source +evidence before answer or abstention. +""" +from __future__ import annotations + +import hashlib +from pathlib import Path +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest +from stage5k102_persistent_typed_discourse_agent import ( + IndexedTypedDiscourseJournal, + PersistentTypedDiscourseAgent, + TypedDiscourseState, + _K102_TURN, + _trim, +) + + +SCHEMA = "oracle-stage5k104-relational-ingress-equivalence-agent-v1" + + +class RelationalIngressEquivalenceAgent(PersistentTypedDiscourseAgent): + """K102 plus generalized speech-act and epistemic-ingress equivalence.""" + + _DIRECT_RHETORICAL = re.compile( + r"^(?:please\s+)?(outline|structure|plan|draft)\s+(a|an|the)\s+" + r"(paper|essay|article)\s+(arguing\s+that|about|on)\s+(.+?)[?.!]*$", + re.IGNORECASE, + ) + _AUTHORITATIVE_SCIENCE = re.compile( + r"^(?:please\s+)?(?:give|provide|state|explain|describe)(?:\s+me)?\s+" + r"(?:the\s+)?(?:(?:accepted|established|recognized)\s+)?" + r"(?:scientific\s+)(definition|mechanism|account|explanation)\s+" + # Longest determiner first: otherwise the ``a`` alternative consumes + # the first character of ``an`` and corrupts the referent boundary. + r"(?:of|for)\s+(?:the|an|a)?\s*(.+?)[?.!]*$", + re.IGNORECASE, + ) + + def __init__( + self, + *args: Any, + discourse_journal_path: Path | None = None, + k104_ablations: Sequence[str] = (), + **kwargs: Any, + ) -> None: + substrate_path = Path(args[5] if len(args) >= 6 else kwargs["substrate_path"]) + self._k104_ablations = frozenset(clean(value).casefold() for value in k104_ablations) + journal_path = discourse_journal_path or substrate_path.with_suffix(".k104.discourse.sqlite3") + super().__init__(*args, discourse_journal_path=journal_path, **kwargs) + # The successor journal is bound to the successor implementation, + # rather than silently inheriting K102's module identity. + self.discourse_journal = IndexedTypedDiscourseJournal( + journal_path, + source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + ) + self._k104_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k102_lineage_sha256, + "opened_failure_source": "stage5k103_untouched_phase_transition_report_v1", + "repairs": [ + "direct_and_periphrastic_rhetorical_imperative_equivalence", + "authoritative_scientific_exact_referent_source_gate", + "unsupported_scientific_referent_abstention", + ], + }) + + @staticmethod + def _exact_referent_present(referent: str, fact: str) -> bool: + normalized_referent = " ".join(re.findall(r"[a-z0-9]+", clean(referent).casefold())) + normalized_fact = " ".join(re.findall(r"[a-z0-9]+", clean(fact).casefold())) + return bool(normalized_referent and re.search( + rf"(?:^|\s){re.escape(normalized_referent)}(?:\s|$)", normalized_fact, + )) + + def _scientific_ingress_result(self, text: str) -> HandlerResult | None: + match = self._AUTHORITATIVE_SCIENCE.match(clean(text)) + if match is None: + return None + relation, referent = match.group(1).casefold(), _trim(match.group(2)) + if "authoritative_scientific_referent_gate" in self._k104_ablations: + return None + terms = tuple(dict.fromkeys(re.findall(r"[a-z0-9]+", referent.casefold()))) + rows = () + if terms: + query = " AND ".join(f'"{term}"' for term in terms) + rows = self.information_source_expert.connection.execute( + "SELECT text_sha256, fact, source_document, source_document_sha256, " + "domain_id, license_note, source_split, bm25(source_rows) " + "FROM source_rows WHERE source_rows MATCH ? ORDER BY bm25(source_rows) LIMIT 128", + (query,), + ).fetchall() + authoritative = [] + for row in rows: + source, domain, fact = str(row[2]), str(row[4]), clean(row[1]) + admitted_authority = source.startswith("openstax/") or domain in { + "life_health", "physical_engineering", "earth_space", "social_behavioral", + "governance_law", "economics_organization", + } + if ( + admitted_authority + and self._exact_referent_present(referent, fact) + and not fact.rstrip().endswith("?") + ): + authoritative.append(row) + current = _K102_TURN.get() + if current is not None: + current.audit.update({ + "recognized": True, + "discourse_operation": "authoritative_scientific_referent_gate", + "requested_relation": relation, + "referent": referent, + "exact_authoritative_rows": len(authoritative), + }) + if not authoritative: + return self._abstention( + "unsupported_scientific_referent", + f"an established scientific {relation} for {referent}", + "exact_authoritative_referent_evidence_absent", + ) + row = authoritative[0] + fact = _trim(row[1]) + return HandlerResult( + "ground_authoritative_scientific_relation", "answered", + (("relation", relation), ("referent", referent)), + (("source_rows", 1),), + ( + SemanticClause( + f"The authenticated source on {referent}", "state", object_value=fact, + ), + ), + ((str(row[3]), str(row[0])),), + ( + "recognize_authoritative_scientific_request", + "bind_complete_multiword_referent", + "require_exact_referent_in_authenticated_source", + "release_supported_relation_at_malkuth", + ), + tool_audit={ + "tool": "k104_exact_scientific_referent_gate", + "source_document": str(row[2]), + "license_note": str(row[5]), + "read_only": True, + "traditional_language_model": False, + }, + ) + + def _paper_result(self, state: TypedDiscourseState, text: str) -> HandlerResult | None: + direct = self._DIRECT_RHETORICAL.match(clean(text)) + if direct is not None and "direct_rhetorical_equivalence" not in self._k104_ablations: + verb, article, genre, relation, claim = ( + _trim(direct.group(index)) for index in range(1, 6) + ) + canonical = f"Help me {verb} {article} {genre} {relation} {claim}." + result = super()._paper_result(state, canonical) + current = _K102_TURN.get() + if current is not None: + current.audit.update({ + "recognized": result is not None, + "discourse_operation": "direct_rhetorical_equivalence", + "canonical_surface_sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), + "meaning_preserved": True, + }) + if result is not None: + return HandlerResult( + result.operation, result.status, result.arguments, result.result, + result.clauses, result.evidence, + ( + "recognize_direct_rhetorical_imperative", + "commute_politeness_periphrasis_without_changing_roles", + *result.computation_trace, + ), + tool_audit=result.tool_audit, memory_audit=result.memory_audit, + ) + return super()._paper_result(state, text) + + def _handle_evidence(self, text: str, context: Mapping[str, Any]) -> HandlerResult | None: + scientific = self._scientific_ingress_result(text) + return scientific if scientific is not None else super()._handle_evidence(text, context) + + def _handle_event(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + scientific = self._scientific_ingress_result(text) + return scientific if scientific is not None else super()._handle_event(session_id, text, turn_id) + + def _event_capability_family(self, text: str) -> tuple[str, ...]: + if self._DIRECT_RHETORICAL.match(clean(text)) or self._AUTHORITATIVE_SCIENCE.match(clean(text)): + return ( + "linguistic", "synthesis", "epistemic", "contrast", "causal", + "mechanism", "tool_router", "dataset", "provenance", + ) + return super()._event_capability_family(text) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + output = super().answer(*args, **kwargs) + audit = (output.get("typed_discourse_workspace") or {}).get("audit") or {} + output["relational_ingress_equivalence"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k104_lineage_sha256, + "opened_failure_source": "immutable_k103_one_shot_failure", + "direct_rhetorical_equivalence_activated": audit.get("discourse_operation") + == "direct_rhetorical_equivalence", + "scientific_referent_gate_activated": audit.get("discourse_operation") + == "authoritative_scientific_referent_gate", + "exact_authoritative_rows": int(audit.get("exact_authoritative_rows", 0)), + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"].update({ + "relational_ingress_equivalence_model_calls": 0, + "relational_ingress_equivalence_gradient_steps": 0, + }) + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["RelationalIngressEquivalenceAgent", "SCHEMA"] diff --git a/root/wander-train/stage5k108_universal_terminal_proof_projection_agent.py b/root/wander-train/stage5k108_universal_terminal_proof_projection_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..b9fb71ba6006f48b63f40a16cdd1430f3e4a79b6 --- /dev/null +++ b/root/wander-train/stage5k108_universal_terminal_proof_projection_agent.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""K108 universal terminal-proof projection successor. + +K106 and the immutable K107 one-shot evidence remain untouched. K107 +prospectively confirmed scientific proof-ledger congruence, while an unrelated +causal turn exposed the same telemetry mismatch outside the scientific gate. +This successor applies the already causal rule universally: the public +knowledge ledger contains exactly those authenticated public rows referenced +by the terminal typed proof, irrespective of upstream candidates. +""" +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any, Sequence + +from stage5k15_native_mixed_agent import clean, digest +from stage5k102_persistent_typed_discourse_agent import IndexedTypedDiscourseJournal +from stage5k106_terminal_proof_provenance_agent import TerminalProofProvenanceAgent + + +SCHEMA = "oracle-stage5k108-universal-terminal-proof-projection-agent-v1" + + +class UniversalTerminalProofProjectionAgent(TerminalProofProvenanceAgent): + """Make terminal proof evidence authoritative for every public ledger.""" + + def __init__( + self, + *args: Any, + discourse_journal_path: Path | None = None, + k108_ablations: Sequence[str] = (), + **kwargs: Any, + ) -> None: + substrate_path = Path(args[5] if len(args) >= 6 else kwargs["substrate_path"]) + self._k108_ablations = frozenset(clean(value).casefold() for value in k108_ablations) + journal_path = discourse_journal_path or substrate_path.with_suffix(".k108.discourse.sqlite3") + super().__init__(*args, discourse_journal_path=journal_path, **kwargs) + self.discourse_journal = IndexedTypedDiscourseJournal( + journal_path, + source_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + ) + self._k108_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k106_lineage_sha256, + "opened_failure_source": "stage5k107_untouched_proof_congruence_report_v1", + "repair": "universal_terminal_typed_proof_projection_to_public_knowledge_ledger", + }) + + def _universal_projection(self, output: dict[str, Any]) -> dict[str, Any]: + if "universal_terminal_proof_projection" in self._k108_ablations: + return {"activated": False, "ablated": True} + proof = output.get("computation_proof") or {} + admitted_rows = [] + seen = set() + for pair in proof.get("evidence", ()): + if not isinstance(pair, (list, tuple)) or len(pair) != 2: + continue + source_sha, text_sha = str(pair[0]), str(pair[1]) + if len(source_sha) != 64 or len(text_sha) != 64 or text_sha in seen: + continue + row = self._source_row_for_proof_evidence(text_sha) + if row is None or row["source_document_sha256"] != source_sha: + continue + admitted_rows.append(row); seen.add(text_sha) + lattice = dict(output.get("knowledge_lattice") or {}) + prior_rows = tuple(lattice.get("evidence_rows_admitted_to_proof") or ()) + prior_admitted = bool(lattice.get("retrieval_admitted_to_proof")) + admitted = bool(admitted_rows) + lattice.update({ + "candidate_retrieval_observed": bool(prior_rows), + "candidate_rows_before_universal_projection": len(prior_rows), + "candidate_admission_before_universal_projection": prior_admitted, + "evidence_rows_admitted_to_proof": admitted_rows, + "retrieval_admitted_to_proof": admitted, + "retrieval_gate_reason": ( + "terminal_typed_proof_contains_authenticated_public_source" + if admitted else "terminal_typed_proof_contains_no_authenticated_public_source" + ), + "relational_provenance_reconciled": True, + "terminal_proof_authoritative_for_admission": True, + "universal_terminal_proof_projection": True, + "terminal_proof_operation": str(output.get("operation", "")), + "terminal_proof_status": str(output.get("status", "")), + "terminal_proof_evidence_sha256s": [row["text_sha256"] for row in admitted_rows], + }) + output["knowledge_lattice"] = lattice + return { + "activated": True, + "ablated": False, + "operation": str(output.get("operation", "")), + "status": str(output.get("status", "")), + "candidate_admission_before": prior_admitted, + "candidate_rows_before": len(prior_rows), + "terminal_authenticated_rows": len(admitted_rows), + "retrieval_admitted_to_proof": admitted, + "projection_changed_admission": prior_admitted is not admitted, + "proof_and_ledger_congruent": bool( + admitted == bool(admitted_rows) + and {row["text_sha256"] for row in admitted_rows} == seen + ), + } + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + output = super().answer(*args, **kwargs) + projection = self._universal_projection(output) + output["universal_terminal_proof_projection"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k108_lineage_sha256, + "opened_failure_source": "immutable_k107_one_shot_failure", + **projection, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"].update({ + "universal_terminal_proof_projection_model_calls": 0, + "universal_terminal_proof_projection_gradient_steps": 0, + }) + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["UniversalTerminalProofProjectionAgent", "SCHEMA"] diff --git a/root/wander-train/stage5k115_copular_definition_equivalence_agent.py b/root/wander-train/stage5k115_copular_definition_equivalence_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..0a07ab51b6803737afbe623f1107347ddd6f3575 --- /dev/null +++ b/root/wander-train/stage5k115_copular_definition_equivalence_agent.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""K115 explicit copular-definition equivalence over frozen K112. + +The immutable K114 prospective failure showed that K112 could locate an exact, +authoritative row and verify definition direction, yet reject ``X is the +process of ...`` because its deliberately small morphological normalization +mapped ``process`` to ``proces`` before relation-cue comparison. This +successor treats an exact referent followed by a copula and a finite +definitional complement class as the definition relation itself. It contains +no domain entities, answer text, language model, embedding model, or scan. +""" +from __future__ import annotations + +from dataclasses import replace +import hashlib +import re +from pathlib import Path +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import clean, digest +from stage5k81_information_weighted_source_expert import terms +from stage5j1_native_contextual_sense_lattice import WORD +from stage5k112_relation_conditioned_source_agent import ( + RelationConditionedSourceAgent, + RelationRequest, + SourceCertificate, +) + + +SCHEMA = "oracle-stage5k115-copular-definition-equivalence-agent-v1" + +DEFINITIONAL_COMPLEMENT_CLASSES = frozenset({ + "act", "action", "activity", "approach", "condition", "concept", + "doctrine", "event", "form", "framework", "idea", "means", "method", + "entity", "operation", "orientation", "pattern", "phenomenon", "practice", + "principle", "procedure", "process", "property", "relation", "rule", + "state", "step", "steps", "system", "technique", "theory", "type", "way", +}) + +NON_DEFINITIONAL_COMPLEMENT_MODIFIERS = frozenset({ + "best", "commonly", "correct", "different", "important", "less", "more", + "most", "not", "now", "only", "sometimes", "used", +}) + + +def _ordered_phrase(value: str) -> str: + """Normalize surface tokens without erasing repetition or order.""" + normalized = [] + for raw in WORD.findall(str(value)): + token = raw.casefold().replace("’", "'") + if token.endswith("'s") and len(token) > 2: + token = token[:-2] + normalized.append(token) + return " ".join(normalized) + + +class CopularDefinitionEquivalenceAgent(RelationConditionedSourceAgent): + """Admit explicit copular definitions missed by K112 normalization.""" + + def __init__(self, *args: Any, k115_ablations: Sequence[str] = (), **kwargs: Any) -> None: + self._k115_ablations = frozenset(clean(value).casefold() for value in k115_ablations) + super().__init__(*args, **kwargs) + self._k115_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k112_lineage_sha256, + "preserved_failure_source": "immutable_stage5k114_untouched_relational_transition_report_v1", + "repair": "exact_referent_copula_definitional_complement_equivalence", + }) + + @classmethod + def _relation_request(cls, text: str) -> RelationRequest | None: + request = super()._relation_request(text) + if request is None or request.relation != "definition": + return request + # K112's frozen optional-article alternation checks ``a`` before + # ``an`` and can therefore consume only the first character of ``an``. + # Reconstruct the definition referent from the original surface with + # longest-article-first precedence. The request's already derived + # content/referent terms remain valid and contain no answer knowledge. + value = clean(text).rstrip(".?!") + match = re.match( + r"^(?:what\s+(?:is|are)|define)\s+(?:(?:an|the|a)\s+)?(.+)$", + value, re.IGNORECASE, + ) + if match is None: + return request + referent = clean(match.group(1)) + return replace(request, referent=referent) if referent else request + + @staticmethod + def _copular_definition_match(request: RelationRequest, fact: str) -> Mapping[str, Any] | None: + if request.relation != "definition": + return None + normalized_fact = _ordered_phrase(fact) + normalized_referent = _ordered_phrase(request.referent) + if not normalized_referent: + return None + pattern = re.compile( + rf"^(?:(?:a|an|the)\s+)?{re.escape(normalized_referent)}\s+" + rf"(?Pis|are|means|denotes|describes|refers\s+to)\s+" + rf"(?:(?:a|an|the)\s+)?(?P[a-z-]+(?:\s+[a-z-]+){{0,3}})\b", + ) + match = pattern.search(normalized_fact) + if match is None: + return None + phrase = match.group("complement_phrase").split() + head_index = next( + (index for index, value in enumerate(phrase) if value in DEFINITIONAL_COMPLEMENT_CLASSES), + None, + ) + if head_index is None: + return None + modifiers = phrase[:head_index] + if any(value in NON_DEFINITIONAL_COMPLEMENT_MODIFIERS for value in modifiers): + return None + return { + "operator": "copular_definition", + "copula": match.group("copula"), + "complement_class": phrase[head_index], + "complement_modifiers": modifiers, + "exact_referent_prefix": True, + } + + def _copular_certificate( + self, request: RelationRequest, parent: SourceCertificate, + ) -> SourceCertificate | None: + if request.relation != "definition": + return None + # Address with the preserved referent surface as well as variants. + # K112's request lemmas are useful for broad relational matching but + # intentionally lossy (for example ``process`` -> ``proces`` and + # ``retailing`` -> ``retail``); exact copular definitions require the + # original noun surface to remain available to FTS. + referent_surfaces = tuple( + value for value in terms(request.referent) + if self.information_source_expert._fts_token(value) + ) + values = set(referent_surfaces) + if not values: + return None + referent_expression = " AND ".join(f'"{value}"' for value in referent_surfaces) + relation_expression = " OR ".join( + f'"{value}"' for value in sorted(DEFINITIONAL_COMPLEMENT_CLASSES) + ) + expression = f"{referent_expression} AND ({relation_expression})" + rows = self.information_source_expert.connection.execute( + "SELECT text_sha256, fact, source_document, source_document_sha256, " + "domain_id, license_note, source_split, bm25(source_rows) " + "FROM source_rows WHERE source_rows MATCH ? ORDER BY bm25(source_rows) LIMIT 768", + (expression,), + ).fetchall() + ranked = [] + audits = [] + for row in rows: + fact = clean(row[1]) + match = self._copular_definition_match(request, fact) + noise, noise_reasons = self._row_noise(fact) + source_name = str(row[2]) + authority = 5.0 if source_name.startswith("openstax/") else 1.5 if source_name.startswith("gutenberg/") else 0.0 + declarative = not fact.rstrip().endswith("?") and "multiple_choice_source" not in noise_reasons + eligible = bool(match is not None and declarative and authority > 0) + score = 20.0 + authority - noise + (2.0 if match and match["copula"] in {"is", "are"} else 0.0) + audit = { + "text_sha256": str(row[0]), "source_document": source_name, + "eligible": eligible, "operator_match": match, + "declarative": declarative, "authority": authority, + "noise_reasons": list(noise_reasons), "score": score, + } + audits.append((score, float(row[7]), str(row[0]), audit)) + if eligible: + ranked.append((score, float(row[7]), str(row[0]), row, audit)) + audits.sort(key=lambda value: (-value[0], value[1], value[2])) + if not ranked: + return None + ranked.sort(key=lambda value: (-value[0], value[1], value[2])) + best = ranked[0] + row = best[3] + selected = { + "text_sha256": str(row[0]), "fact": clean(row[1]), + "source_document": str(row[2]), "source_document_sha256": str(row[3]), + "domain_id": str(row[4]), "license_note": str(row[5]), + "source_split": str(row[6]), + "admission_authority": "k115_copular_definition_equivalence", + } + runner = ranked[1][0] if len(ranked) > 1 else None + return SourceCertificate( + request=request, selected_row=selected, accepted=True, + reason="copular_definition_equivalence_satisfied", + selected_score=best[0], runner_up_score=runner, + inspected_postings=len(rows), eligible_candidates=len(ranked), + audit_candidates=tuple(value[3] for value in audits[:5]), + ) + + def _certificate(self, request: RelationRequest) -> SourceCertificate: + parent = super()._certificate(request) + if "copular_definition_equivalence" in self._k115_ablations: + return parent + # An exact sentence-prefix copular definition is a stronger relation + # certificate than an embedded mention admitted by K112's broad + # lexical relation cues. Prefer it even when the parent found a weaker + # candidate; other relation kinds remain completely unchanged. + return self._copular_certificate(request, parent) or parent + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + output = super().answer(*args, **kwargs) + certificate = output.get("relation_conditioned_source_certificate") or {} + activated = certificate.get("reason") == "copular_definition_equivalence_satisfied" + output["copular_definition_equivalence"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k115_lineage_sha256, + "activated": activated, + "ablated": "copular_definition_equivalence" in self._k115_ablations, + "operator": "exact_referent_plus_copula_plus_definitional_complement_class", + "definitional_complement_class_count": len(DEFINITIONAL_COMPLEMENT_CLASSES), + "maximum_complement_modifiers": 3, + "domain_entities_encoded": 0, + "traditional_language_model": False, "transformer": False, + "pretrained_embedding_encoder": False, "autoregressive_carrier": False, + "model_calls": 0, "gradient_steps": 0, + } + output["runtime"].update({ + "copular_definition_equivalence_model_calls": 0, + "copular_definition_equivalence_gradient_steps": 0, + }) + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["CopularDefinitionEquivalenceAgent", "SCHEMA", "DEFINITIONAL_COMPLEMENT_CLASSES"] diff --git a/root/wander-train/stage5k117_caller_context_isolation_agent.py b/root/wander-train/stage5k117_caller_context_isolation_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..08e3f8f62cad996bd7859f9fe784d5babf0e68e8 --- /dev/null +++ b/root/wander-train/stage5k117_caller_context_isolation_agent.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""K117 caller-context isolation over immutable K115/K116 evidence. + +K116 established that K115's relation operator transfers to fresh definitions, +but exposed an independent precedence defect on a causal assertion. K90 may +insert a private retrieval sentinel below the outer answer boundary. K81 saw +that sentinel as though it were caller-supplied evidence and consequently +demoted an already valid ``assert_causal`` graph to ``delegate``. The +misrouted ephemeral event then made deterministic restart replay correctly +reject the changed operation. + +This successor separates caller context from internal carrier context. A +typed graph that consumes the utterance remains prior to private sentinels; +genuine caller-supplied evidence, corpus records, and sensor streams retain +their bounded rails. No domain words, answer strings, model, embedding +encoder, gradient update, or autoregressive carrier are introduced. +""" +from __future__ import annotations + +from collections.abc import Sequence +from contextvars import ContextVar +from typing import Any, Mapping + +from stage5k15_native_mixed_agent import clean, digest +from stage5k81_native_relational_construction_agent import _CONTEXT_KIND +from stage5k102_persistent_typed_discourse_agent import K102FinitePredicateTransducer +from stage5k115_copular_definition_equivalence_agent import ( + CopularDefinitionEquivalenceAgent, +) + + +SCHEMA = "oracle-stage5k117-caller-context-isolation-agent-v1" + +_K117_CALLER_CONTEXT_KIND: ContextVar[str] = ContextVar( + "k117_caller_context_kind", default="none", +) +_K117_ISOLATION_AUDIT: ContextVar[Mapping[str, Any] | None] = ContextVar( + "k117_isolation_audit", default=None, +) + + +def _caller_context_kind(context: Mapping[str, Any] | None) -> str: + value = context or {} + for key in ("evidence", "training_corpus", "sensor_stream"): + payload = value.get(key) + if isinstance(payload, Sequence) and not isinstance(payload, (str, bytes)) and payload: + return key + return "none" + + +class CallerContextIsolationTransducer(K102FinitePredicateTransducer): + """Make graph precedence depend on the outer caller boundary.""" + + def analyze(self, text: str, discourse, *, known_frames=()): + inherited_context_kind = _CONTEXT_KIND.get() + caller_context_kind = _K117_CALLER_CONTEXT_KIND.get() + ablated = "caller_context_isolation" in self.owner._k117_ablations + + # Obtain the fully repaired inherited graph with context influence + # suppressed only when context was introduced internally. This calls + # through K102/K94/.../K81's MRO once and retains every later parser + # repair. For genuine caller context the inherited behavior is exact. + token = None + if not ablated and caller_context_kind == "none" and inherited_context_kind != "none": + token = _CONTEXT_KIND.set("none") + try: + graph = super().analyze(text, discourse, known_frames=known_frames) + finally: + if token is not None: + _CONTEXT_KIND.reset(token) + + _K117_ISOLATION_AUDIT.set({ + "caller_context_kind": caller_context_kind, + "inherited_context_kind": inherited_context_kind, + "internal_context_isolated": token is not None, + "ablated": ablated, + "selected_speech_act": graph.speech_act, + "typed_graph_consumes_utterance": bool(graph.consumes_as_graph), + }) + return graph + + +class CallerContextIsolationAgent(CopularDefinitionEquivalenceAgent): + """Preserve K115 while preventing private carrier/context confusion.""" + + def __init__(self, *args: Any, k117_ablations: Sequence[str] = (), **kwargs: Any) -> None: + self._k117_ablations = frozenset(clean(value).casefold() for value in k117_ablations) + super().__init__(*args, **kwargs) + self.graph_transducer = CallerContextIsolationTransducer( + self, self.construction_head, + ) + self._k117_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k115_lineage_sha256, + "preserved_failure_source": ( + "immutable_stage5k116_untouched_operator_transfer_report_v1" + ), + "repair": "outer_caller_context_distinct_from_internal_retrieval_carrier", + }) + + def _handle_evidence(self, text: str, context: Mapping[str, Any]): + evidence = context.get("evidence") + private_carrier = bool( + isinstance(evidence, Sequence) + and not isinstance(evidence, (str, bytes)) + and any( + isinstance(value, Mapping) + and str(value.get("answer_status", "")).startswith("k90_") + for value in evidence + ) + ) + graph = self._k76_current.get() + typed_graph_prior = bool( + _K117_CALLER_CONTEXT_KIND.get() == "none" + and "caller_context_isolation" not in self._k117_ablations + and private_carrier + and graph is not None + and graph.speech_act in self.DIRECT_ACTS + and graph.consumes_as_graph + ) + if typed_graph_prior: + audit = dict(_K117_ISOLATION_AUDIT.get() or {}) + audit.update({ + "private_carrier_detected": True, + "private_carrier_bypassed_evidence_handler": True, + "typed_graph_precedence_preserved": True, + }) + _K117_ISOLATION_AUDIT.set(audit) + return None + return super()._handle_evidence(text, context) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + caller_kind = _caller_context_kind(context) + caller_token = _K117_CALLER_CONTEXT_KIND.set(caller_kind) + audit_token = _K117_ISOLATION_AUDIT.set(None) + try: + output = super().answer(session_id, text, context, **kwargs) + audit = dict(_K117_ISOLATION_AUDIT.get() or {}) + finally: + _K117_ISOLATION_AUDIT.reset(audit_token) + _K117_CALLER_CONTEXT_KIND.reset(caller_token) + output["caller_context_isolation"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k117_lineage_sha256, + "development_source": "opened_immutable_k116_failure", + "prospective_claim": False, + "caller_context_kind": caller_kind, + "ablated": "caller_context_isolation" in self._k117_ablations, + "audit": audit, + "private_retrieval_sentinel_is_caller_evidence": False, + "genuine_caller_context_precedence_preserved": True, + "domain_entities_encoded": 0, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"].update({ + "caller_context_isolation_model_calls": 0, + "caller_context_isolation_gradient_steps": 0, + }) + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["CallerContextIsolationAgent", "CallerContextIsolationTransducer", "SCHEMA"] diff --git a/root/wander-train/stage5k11_quadratic_inverse_solver.py b/root/wander-train/stage5k11_quadratic_inverse_solver.py new file mode 100644 index 0000000000000000000000000000000000000000..d0b3381fe4f4a65baad559a3469956fdbc380920 --- /dev/null +++ b/root/wander-train/stage5k11_quadratic_inverse_solver.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +"""Stage 5K11 proof-conditioned quadratic native inverse solver. + +The solver chooses a complete construction coordinate in one global decision. +It never predicts a next token. Candidate clauses are compiled from the typed +computation proof, checked by the native forward analyzer, and represented as +binary one-hot variables in an auditable QUBO: + + E(x) = sum_i u_i x_i + P (sum_i x_i - 1)^2 + +The unary semantic energy ``u_i`` is itself decomposed into frame, operator, +role, alpha-equivalence, and minimal-chart-complexity terms. This initial +whole-construction QUBO is an exact verifier and a bridge to the later +factorized role/slot/discourse QUBO; it already replaces serial surface search +with a globally selected sentence-shaped object. +""" + +from __future__ import annotations + +import hashlib +import itertools +from dataclasses import dataclass +from typing import Any, Mapping, Sequence + +import numpy as np + +from stage5k3_language_adapter import LanguageAdapter +from stage5k10_self_inverse_computation import ( + NativeEnglishInverseGrammar, + NativeRealization, + RoundTripAudit, + SelfInverseProof, + digest, +) + + +SCHEMA = "oracle-stage5k11-quadratic-inverse-solver-v1" +CONSTRUCTION_VARIANTS = ( + "frame_order", + "nominals_before_prepositions", + "prepositions_before_nominals", +) + + +@dataclass(frozen=True) +class QuadraticCandidate: + candidate_id: str + realization: NativeRealization + round_trip: RoundTripAudit + role_transformations: tuple[tuple[str, str], ...] + preposition_choices: tuple[tuple[str, str], ...] + energy_terms: tuple[tuple[str, float], ...] + + @property + def unary_energy(self) -> float: + return float(sum(value for _, value in self.energy_terms)) + + def as_dict(self) -> dict[str, Any]: + return { + "candidate_id": self.candidate_id, + "realization": self.realization.as_dict(), + "round_trip": self.round_trip.as_dict(), + "role_transformations": [ + [role, hashlib.sha256(surface.casefold().encode("utf-8")).hexdigest()] + for role, surface in self.role_transformations + ], + "preposition_choices": [list(value) for value in self.preposition_choices], + "energy_terms": [list(value) for value in self.energy_terms], + "unary_energy": self.unary_energy, + } + + +@dataclass(frozen=True) +class QuadraticInverseResolution: + proof_sha256: str + selected: QuadraticCandidate + candidates: tuple[QuadraticCandidate, ...] + assignment: tuple[int, ...] + penalty: float + constant: float + quadratic_matrix_sha256: str + quadratic_energy: float + exhaustive_one_hot_minimum_verified: bool + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA, + "proof_sha256": self.proof_sha256, + "selected": self.selected.as_dict(), + "candidate_count": len(self.candidates), + "candidate_manifest_sha256": digest([ + value.as_dict() for value in self.candidates + ]), + "assignment": list(self.assignment), + "assignment_hamming_weight": sum(self.assignment), + "penalty": self.penalty, + "constant": self.constant, + "quadratic_matrix_sha256": self.quadratic_matrix_sha256, + "quadratic_energy": self.quadratic_energy, + "exhaustive_one_hot_minimum_verified": ( + self.exhaustive_one_hot_minimum_verified + ), + "whole_construction_selected_globally": True, + "next_token_predictions": 0, + "traditional_language_model": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "global_gradient_steps": 0, + } + + +class QuadraticInverseSolver: + def __init__( + self, + grammar: NativeEnglishInverseGrammar, + adapter: LanguageAdapter, + *, + maximum_hypotheses: int = 256, + maximum_candidates: int = 128, + one_hot_penalty: float = 1024.0, + ) -> None: + self.grammar = grammar + self.adapter = adapter + self.maximum_hypotheses = max(1, int(maximum_hypotheses)) + self.maximum_candidates = max(1, int(maximum_candidates)) + self.one_hot_penalty = float(one_hot_penalty) + + def _role_options(self, proof: SelfInverseProof) -> tuple[ + tuple[str, tuple[str, ...]], ... + ]: + frame = self.grammar.frames[proof.frame_id] + subject_roles = { + role for slot, role in frame.surface_slots if slot == "subject" + } + pronouns = { + "i", "me", "we", "us", "you", "he", "him", "she", "her", + "it", "they", "them", "who", "whom", "someone", "something", + } + result: list[tuple[str, tuple[str, ...]]] = [] + for filler in proof.role_fillers: + raw = filler.surface + variants = [raw] + capitalized = raw[:1].upper() + raw[1:] if raw else raw + if capitalized != raw: + variants.append(capitalized) + if " and " in raw.casefold() or " or " in raw.casefold(): + coordinated = " ".join( + value if value.casefold() in {"and", "or"} + else value[:1].upper() + value[1:] + for value in raw.split() + ) + if coordinated not in variants: + variants.append(coordinated) + # A sentence-initial nominal homograph (stick, knife, storm) can be + # misread as the predicate. The determiner is a pure construction + # coordinate: forward analysis recovers the same role referent. + if ( + filler.role in subject_roles + and raw.casefold() not in pronouns + and " " not in raw.strip() + ): + variants.append("the " + raw) + result.append((filler.role, tuple(variants))) + return tuple(result) + + def _preposition_options( + self, proof: SelfInverseProof, + ) -> tuple[tuple[str, tuple[str, ...]], ...]: + frame = self.grammar.frames[proof.frame_id] + return tuple( + (slot, self.grammar._preposition_candidates(slot)) + for slot, _ in frame.surface_slots if slot.startswith("prep:") + ) + + @staticmethod + def _products( + values: Sequence[tuple[str, tuple[str, ...]]], + ) -> tuple[dict[str, str], ...]: + if not values: + return ({},) + keys = tuple(value[0] for value in values) + return tuple( + dict(zip(keys, choices)) + for choices in itertools.product(*(value[1] for value in values)) + ) + + @staticmethod + def _energy_terms( + audit: RoundTripAudit, + *, + role_transformations: Sequence[tuple[str, str]], + preposition_choices: Sequence[tuple[str, str]], + construction_variant_id: str, + ) -> tuple[tuple[str, float], ...]: + # Semantic invariants dominate chart simplicity by several orders of + # magnitude. The small final terms only break semantically equal ties. + return ( + ("frame_mismatch", 128.0 * float(not audit.frame_recovered)), + ( + "family_mismatch", + 64.0 * float(not audit.transformation_family_recovered), + ), + ( + "operator_sequence_mismatch", + 64.0 * float(not audit.operator_sequence_recovered), + ), + ( + "known_role_mismatch", + 32.0 * float(not audit.role_fillers_recovered), + ), + ( + "alpha_equivalence_mismatch", + 16.0 * float(not audit.semantic_round_trip_alpha_equivalent), + ), + ( + "strict_identity_residual", + 0.125 * float(not audit.semantic_round_trip_exact), + ), + ("surface_coordinate_changes", 0.001 * len(role_transformations)), + ("preposition_coordinate_changes", 0.001 * len(preposition_choices)), + ( + "construction_order_change", + 0.001 * float(construction_variant_id != "frame_order"), + ), + ) + + @staticmethod + def _matrix( + unary: Sequence[float], penalty: float, + ) -> tuple[np.ndarray, float]: + count = len(unary) + matrix = np.full((count, count), float(penalty), dtype=np.float64) + np.fill_diagonal( + matrix, + np.asarray(unary, dtype=np.float64) - float(penalty), + ) + return matrix, float(penalty) + + @staticmethod + def _quadratic_energy( + assignment: np.ndarray, matrix: np.ndarray, constant: float, + ) -> float: + return float(assignment @ matrix @ assignment + constant) + + def solve(self, proof: SelfInverseProof) -> QuadraticInverseResolution: + role_products = self._products(self._role_options(proof)) + prep_products = self._products(self._preposition_options(proof)) + candidates: list[QuadraticCandidate] = [] + seen_surfaces: set[str] = set() + original_roles = proof.role_map + default_preps = { + key: self.grammar._preposition(key) + for key, _ in self._preposition_options(proof) + } + for construction_variant_id, role_values, prep_values in itertools.product( + CONSTRUCTION_VARIANTS, role_products, prep_products, + ): + realization = self.grammar.realize( + proof, + role_surface_overrides=role_values, + preposition_overrides=prep_values, + construction_variant_id=construction_variant_id, + ) + # Case is semantically significant to the rule-bounded analyzer, + # so de-duplicate exact surfaces rather than case-folded surfaces. + surface_key = realization.text + if surface_key in seen_surfaces: + continue + seen_surfaces.add(surface_key) + audit = self.grammar.audit_round_trip( + proof, realization, self.adapter, + maximum_hypotheses=self.maximum_hypotheses, + ) + role_changes = tuple(sorted( + (role, surface) + for role, surface in role_values.items() + if role in original_roles + and surface != original_roles[role].surface + )) + prep_changes = tuple(sorted( + (slot, value) + for slot, value in prep_values.items() + if value != default_preps.get(slot) + )) + terms = self._energy_terms( + audit, + role_transformations=role_changes, + preposition_choices=prep_changes, + construction_variant_id=construction_variant_id, + ) + payload: Mapping[str, Any] = { + "proof_sha256": proof.proof_sha256, + "surface_sha256": hashlib.sha256( + realization.text.encode("utf-8") + ).hexdigest(), + "variant": construction_variant_id, + "role_changes": role_changes, + "prep_changes": prep_changes, + } + candidates.append(QuadraticCandidate( + candidate_id="qic:" + digest(payload)[:28], + realization=realization, + round_trip=audit, + role_transformations=role_changes, + preposition_choices=prep_changes, + energy_terms=terms, + )) + if len(candidates) >= self.maximum_candidates: + break + if not candidates: + raise RuntimeError("proof produced no inverse construction candidates") + candidates.sort(key=lambda value: (value.unary_energy, value.candidate_id)) + unary = tuple(value.unary_energy for value in candidates) + matrix, constant = self._matrix(unary, self.one_hot_penalty) + selected_index = 0 + assignment = np.zeros(len(candidates), dtype=np.float64) + assignment[selected_index] = 1.0 + energy = self._quadratic_energy(assignment, matrix, constant) + one_hot_energies = [ + self._quadratic_energy(np.eye(1, len(candidates), index)[0], matrix, constant) + for index in range(len(candidates)) + ] + verified = ( + abs(energy - min(one_hot_energies)) <= 1e-9 + and self.one_hot_penalty > max(unary) + ) + return QuadraticInverseResolution( + proof_sha256=proof.proof_sha256, + selected=candidates[selected_index], + candidates=tuple(candidates), + assignment=tuple(int(value) for value in assignment), + penalty=self.one_hot_penalty, + constant=constant, + quadratic_matrix_sha256=hashlib.sha256( + np.ascontiguousarray(matrix).tobytes() + ).hexdigest(), + quadratic_energy=energy, + exhaustive_one_hot_minimum_verified=verified, + ) + + +__all__ = [ + "CONSTRUCTION_VARIANTS", + "QuadraticCandidate", + "QuadraticInverseResolution", + "QuadraticInverseSolver", + "SCHEMA", +] diff --git a/root/wander-train/stage5k121_authenticated_context_operator_agent.py b/root/wander-train/stage5k121_authenticated_context_operator_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..71bd7aa4dd864d17b2237b532ca81b163f067687 --- /dev/null +++ b/root/wander-train/stage5k121_authenticated_context_operator_agent.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""K121 authenticated-context operator closure over immutable K119/K120. + +The prospective K120 panel exposed a remaining precedence error rather than a +knowledge error. Caller-supplied evidence reached the evidence handler, but an +open lexical comparison recognized the request surface first and displaced the +authenticated records. The same panel exposed an incomplete equivalence class +for a corpus comparative: ``greater`` was reduced only when an auxiliary HRR +head happened to classify the whole request as comparative. + +This successor treats a non-empty authenticated outer context as a binding +operator. Evidence is interpreted by the bounded evidence kernel before open +surface operators, while corpus comparative morphemes are normalized by their +directional order relation before bounded relational reduction. It adds no +domain facts, answer strings, pretrained representation, autoregressive +carrier, or gradient-trained language component. +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from contextvars import ContextVar +import re +from typing import Any + +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest +from stage5k88_opened_k87_precedence_retrieval_agent import ( + OpenedK87PrecedenceRetrievalAgent, +) +from stage5k117_caller_context_isolation_agent import _K117_CALLER_CONTEXT_KIND +from stage5k119_social_negative_scope_agent import SocialNegativeScopeAgent + + +SCHEMA = "oracle-stage5k121-authenticated-context-operator-agent-v1" +_K121_AUDIT: ContextVar[Mapping[str, Any] | None] = ContextVar( + "k121_context_operator_audit", default=None, +) + + +class AuthenticatedContextOperatorAgent(SocialNegativeScopeAgent): + """Make authenticated context prior to incompatible open-surface operators.""" + + _HIGH_ORDER = re.compile(r"\b(?:greater|greatest)\b", re.IGNORECASE) + _LOW_ORDER = re.compile(r"\b(?:lesser|smallest|fewest)\b", re.IGNORECASE) + + def __init__(self, *args: Any, k121_ablations: Sequence[str] = (), **kwargs: Any) -> None: + self._k121_ablations = frozenset(clean(value).casefold() for value in k121_ablations) + super().__init__(*args, **kwargs) + self._k121_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k119_lineage_sha256, + "preserved_failure_source": "immutable_stage5k120_untouched_operator_closure_report_v1", + "repairs": [ + "authenticated_outer_evidence_precedes_incompatible_open_surface_operator", + "comparative_lexeme_maps_to_directional_order_before_corpus_reduction", + ], + }) + + @staticmethod + def _outer_evidence(context: Mapping[str, Any]) -> bool: + # The outer-boundary context variable is authoritative. Lower layers + # may replace a private K90 sentinel with retrieved rows; inspecting + # only the mutable carrier at that point would misclassify those rows + # as caller evidence and recreate the K116 collision. + if _K117_CALLER_CONTEXT_KIND.get() != "evidence": + return False + evidence = context.get("evidence") + if not isinstance(evidence, Sequence) or isinstance(evidence, (str, bytes)) or not evidence: + return False + return not any( + isinstance(value, Mapping) + and str(value.get("answer_status", "")).startswith("k90_") + for value in evidence + ) + + @staticmethod + def _project_productive_negative_scope( + result: HandlerResult | None, + context: Mapping[str, Any], + ) -> HandlerResult | None: + """Retain K119's productive scope after direct evidence-kernel entry.""" + if result is None or result.operation != "evidence_synthesis" or result.status != "answered": + return result + evidence = context.get("evidence") + if not isinstance(evidence, Sequence) or isinstance(evidence, (str, bytes)): + return result + facts = [ + clean(value.get("fact", value.get("claim", value))) + if isinstance(value, Mapping) else clean(value) + for value in evidence + ] + negative_replication = any( + "replication" in fact.casefold() + and SocialNegativeScopeAgent._NEGATIVE_SCOPE.search(fact) + for fact in facts + ) + existing_negative = any( + clean(clause.subject).casefold() == "the evidence" + and clean(clause.predicate).casefold() == "establish" + and clean(clause.object_value or "").casefold() == "universal superiority" + and clause.polarity == "negative" + for clause in result.clauses + ) + if not negative_replication or existing_negative: + return result + clauses = tuple( + SemanticClause( + clause.subject, clause.predicate, object_value=clause.object_value, + complement="absent" if ( + clean(clause.subject).casefold() == "independent replication" + and clean(clause.predicate).casefold() == "be" + ) else clause.complement, + tense=clause.tense, modal=clause.modal, polarity=clause.polarity, + qualifiers=clause.qualifiers, + ) + for clause in result.clauses + ) + ( + SemanticClause( + "The evidence", "establish", object_value="universal superiority", + polarity="negative", + ), + ) + values = dict(result.result) + values["independent_replication_absent"] = True + audit = dict(_K121_AUDIT.get() or {}) + audit.update({ + "productive_negative_evidence_scope_activated": True, + "negative_scope_surface_count": sum( + bool(SocialNegativeScopeAgent._NEGATIVE_SCOPE.search(fact)) for fact in facts + ), + }) + _K121_AUDIT.set(audit) + return HandlerResult( + result.operation, result.status, result.arguments, tuple(values.items()), + clauses, result.evidence, + (*result.computation_trace, "bind_productive_negative_evidence_scope", "derive_scoped_non_superiority"), + tool_audit=result.tool_audit, memory_audit=result.memory_audit, + ) + + def _handle_evidence(self, text: str, context: Mapping[str, Any]): + if ( + "authenticated_context_precedence" in self._k121_ablations + or not self._outer_evidence(context) + # Constitutional identity and recognized social speech acts are + # relations to the interlocutor, not claims about supplied data. + or self._identity_request(text) + or self._social_act(text) is not None + ): + return super()._handle_evidence(text, context) + + # Enter at the mature bounded evidence kernel. This deliberately + # bypasses later open-conversation surface interpreters, while retaining + # all lower evidence synthesis and epistemic-boundary behavior. + result = OpenedK87PrecedenceRetrievalAgent._handle_evidence(self, text, context) + result = self._project_productive_negative_scope(result, context) + audit = dict(_K121_AUDIT.get() or {}) + audit.update({ + "context_kind": "evidence", + "authenticated_context_precedence_activated": True, + "selected_operation": result.operation if result is not None else None, + }) + _K121_AUDIT.set(audit) + if result is None: + return None + return HandlerResult( + result.operation, result.status, result.arguments, result.result, + result.clauses, result.evidence, + (*result.computation_trace, "bind_authenticated_outer_evidence_before_open_surface_operator"), + tool_audit=result.tool_audit, memory_audit=result.memory_audit, + ) + + def _handle_corpus( + self, + session_id: str, + text: str, + context: Mapping[str, Any], + turn_id: str, + ): + corpus = context.get("training_corpus") + bounded = bool( + isinstance(corpus, Sequence) + and not isinstance(corpus, (str, bytes)) + and corpus + ) + normalized = text + direction = None + if bounded and "comparative_direction_equivalence" not in self._k121_ablations: + if self._HIGH_ORDER.search(text): + normalized = self._HIGH_ORDER.sub("higher", text) + direction = "higher" + elif self._LOW_ORDER.search(text): + normalized = self._LOW_ORDER.sub("lower", text) + direction = "lower" + result = super()._handle_corpus(session_id, normalized, context, turn_id) + if result is None or normalized == text: + return result + audit = dict(_K121_AUDIT.get() or {}) + audit.update({ + "context_kind": "training_corpus", + "comparative_direction_equivalence_activated": True, + "comparative_direction": direction, + "selected_operation": result.operation, + }) + _K121_AUDIT.set(audit) + return HandlerResult( + result.operation, result.status, result.arguments, result.result, + result.clauses, result.evidence, + (*result.computation_trace, "normalize_comparative_lexeme_to_directional_order"), + tool_audit=result.tool_audit, memory_audit=result.memory_audit, + ) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + token = _K121_AUDIT.set({}) + try: + output = super().answer(*args, **kwargs) + audit = dict(_K121_AUDIT.get() or {}) + finally: + _K121_AUDIT.reset(token) + output["authenticated_context_operator"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k121_lineage_sha256, + "development_source": "opened_immutable_k120_failure", + "prospective_claim": False, + "audit": audit, + "authenticated_context_is_binding_operator": True, + "comparative_lexeme_is_directional_relation": True, + "domain_entities_encoded": 0, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"].update({ + "authenticated_context_operator_model_calls": 0, + "authenticated_context_operator_gradient_steps": 0, + }) + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["AuthenticatedContextOperatorAgent", "SCHEMA"] diff --git a/root/wander-train/stage5k131_registered_source_action_ingress_agent.py b/root/wander-train/stage5k131_registered_source_action_ingress_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..89744c957fe52191ad3311b954b5b0b4c73190a0 --- /dev/null +++ b/root/wander-train/stage5k131_registered_source_action_ingress_agent.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""K131 whole-construction ingress for registered-source action clauses. + +Immutable K129 showed that an explicit multi-step registered-database request +could be captured by lexically adjacent public text before the bounded tool +graph saw it. This successor recognizes the conjunction of four typed +relations: registered source, read-only selection verb, numeric extremum, and +dependent record projection. Only that whole construction receives tool +precedence. Its relations are projected into the inherited typed-goal +algebra while the original surface remains the audited goal and the +H1/C43/Tree/Malkuth path executes it. This layer contains no domain rows, +answer values, or response templates. +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from contextvars import ContextVar +import re +from typing import Any + +from stage5k15_native_mixed_agent import HandlerResult, clean, digest +from stage5k128_persistent_tool_reference_agent import PersistentToolReferenceAgent + + +SCHEMA = "oracle-stage5k131-registered-source-action-ingress-agent-v1" +_K131_AUDIT: ContextVar[dict[str, Any] | None] = ContextVar( + "k131_registered_source_action_ingress_audit", default=None, +) + + +class RegisteredSourceActionIngressAgent(PersistentToolReferenceAgent): + """Project explicit registered-source action constructions to typed tools.""" + + _SOURCE = re.compile( + r"\b(?:registered\s+)?(?:earthquake|population)\s+" + r"(?:database|catalog|observations?|dataset)\b", + re.IGNORECASE, + ) + _SELECT = re.compile( + r"\b(?:use|using|consult|query|search|within|from|select|choose|find|" + r"locate|determine|identify)\b", + re.IGNORECASE, + ) + _EXTREMUM = re.compile( + r"\b(?:minimum|smallest|lowest|weakest|maximum|greatest|highest|" + r"largest|strongest)\b", + re.IGNORECASE, + ) + _DEPENDENCY = re.compile( + r"(?:\b(?:then|after(?:wards|\s+that)?|subsequently)\b|[;,])[^.?!]*" + r"\b(?:provide|report|return|give|state|retrieve)\b", + re.IGNORECASE, + ) + _PROJECTION = re.compile( + r"\b(?:chosen|selected|that)\s+(?:event|record)|" + r"\b(?:record|event)(?:'s)?\s+(?:magnitude|place|value|depth)\b", + re.IGNORECASE, + ) + + def __init__(self, *args: Any, k131_ablations: Sequence[str] = (), **kwargs: Any) -> None: + self._k131_ablations = frozenset(clean(value).casefold() for value in k131_ablations) + super().__init__(*args, **kwargs) + self._k131_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k128_lineage_sha256, + "opened_failure_source": "immutable_stage5k129_untouched_persistent_reference_failure", + "operator": ( + "registered_source+read_only_selection+numeric_extremum+" + "dependent_record_projection" + ), + }) + + @classmethod + def _registered_source_action(cls, text: str) -> dict[str, Any] | None: + value = clean(text) + source = cls._SOURCE.search(value) + select = cls._SELECT.search(value) + extremum = cls._EXTREMUM.search(value) + dependency = cls._DEPENDENCY.search(value) + projection = cls._PROJECTION.search(value) + if not all((source, select, extremum, dependency, projection)): + return None + direction = "minimum" if extremum.group(0).casefold() in { + "minimum", "smallest", "lowest", "weakest", + } else "maximum" + domain = "earthquake" if "earthquake" in source.group(0).casefold() else "population" + return { + "whole_construction_recognized": True, + "domain": domain, + "direction": direction, + "registered_source_surface": source.group(0), + "selection_surface": select.group(0), + "extremum_surface": extremum.group(0), + "dependent_projection_surface": projection.group(0), + "surface_sha256": digest(value), + } + + def _project_registered_source_action( + self, text: str, context: Mapping[str, Any], ingress: str, + ) -> HandlerResult | None: + construction = self._registered_source_action(text) + if ( + construction is None + or "registered_source_action_precedence" in self._k131_ablations + ): + return None + session_id = clean(context.get("k102_session_id", "")) + if not session_id: + return None + # The parent controller already owns the typed action algebra, but its + # surface compiler recognizes only a deliberately small set of + # sequential connectives. K131 has already established all of the + # relations required for a goal, so project those relations to a + # canonical *internal* carrier and compile the plan from that carrier. + # Execute and audit the plan against the untouched user surface. This + # is construction transduction, not answer-text rewriting. + domain = ( + "earthquake database" + if construction["domain"] == "earthquake" + else "population database" + ) + metric = "magnitude" if domain == "earthquake database" else "value" + requested_fields = self._requested_fields(text, domain) + projection = " and ".join(requested_fields) + canonical_carrier = ( + f"From the {domain}, find the {construction['direction']} {metric}; " + "then provide the selected record" + + (f" and {projection}." if projection else ".") + ) + plan = self._compile_goal(canonical_carrier) + result = ( + self._execute_goal(session_id, text, plan) + if plan else self._bounded_tool_graph(session_id, text) + ) + audit = _K131_AUDIT.get() + if audit is not None: + audit.update({ + **construction, + "ingress": ingress, + "selected_operation": result.operation, + "selected_status": result.status, + "tool_graph_precedes_open_public_text": True, + "surface_rewritten": False, + "construction_projected_to_typed_goal": bool(plan), + "typed_goal_action_count": len(plan), + "canonical_carrier_sha256": digest(canonical_carrier), + }) + return HandlerResult( + result.operation, result.status, result.arguments, result.result, + result.clauses, result.evidence, + ("recognize_whole_registered_source_action_construction", + "bind_tool_precedence_before_open_public_text", + *result.computation_trace), + tool_audit={ + **dict(result.tool_audit or {}), + "k131_whole_construction_precedence": True, + "k131_surface_rewritten": False, + "k131_construction_projected_to_typed_goal": bool(plan), + }, + memory_audit=result.memory_audit, + ) + + def _handle_evidence( + self, text: str, context: Mapping[str, Any], + ) -> HandlerResult | None: + projected = self._project_registered_source_action(text, context, "evidence") + return projected if projected is not None else super()._handle_evidence(text, context) + + def _handle_definition( + self, session_id: str, text: str, turn_id: str, + ) -> HandlerResult | None: + # Some mixed specialist selections enter the definition rail before + # evidence. The same construction operator must be prior there too. + projected = self._project_registered_source_action( + text, {"k102_session_id": session_id}, "definition", + ) + return projected if projected is not None else super()._handle_definition(session_id, text, turn_id) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + audit: dict[str, Any] = {} + token = _K131_AUDIT.set(audit) + try: + output = super().answer(*args, **kwargs) + finally: + _K131_AUDIT.reset(token) + output["registered_source_action_ingress"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k131_lineage_sha256, + "development_source": "opened_immutable_k129_failure", + "prospective_claim": False, + "audit": audit, + "whole_construction_precedes_local_lexical_retrieval": True, + "answer_values_encoded": 0, + "domain_entities_encoded": 0, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, "gradient_steps": 0, + } + output["runtime"].update({ + "registered_source_action_ingress_model_calls": 0, + "registered_source_action_ingress_gradient_steps": 0, + }) + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["RegisteredSourceActionIngressAgent", "SCHEMA"] diff --git a/root/wander-train/stage5k138_typed_interpretation_candidate.py b/root/wander-train/stage5k138_typed_interpretation_candidate.py new file mode 100644 index 0000000000000000000000000000000000000000..57e346504cdc45cd7700d7801f2ef1714dbaa38b --- /dev/null +++ b/root/wander-train/stage5k138_typed_interpretation_candidate.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""K138 typed interpretation schema for geometric ingress arbitration. + +An InterpretationCandidate is one hypothesis about what an utterance is +doing: a typed family, the geometric resonance of the utterance against the +family's prototype state, the typed feature evidence that supports or +opposes it, and the resulting bid. An ArbitrationRecord is the simultaneous +scoring of every family with an explicit winner, runner-up, and calibrated +margin — the object that replaces first-match handler precedence. + +The schema is deliberately runtime-independent: it imports nothing from the +agent chain so that the shadow arbiter can be evaluated against frozen panel +transcripts without constructing a runtime. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +FAMILIES = ( + "registered_source_action", + "corpus_compile", + "essay_artifact", + "typed_modal", + "causal_discourse", + "lexical_definition", + "discourse_assertion", + "social", + "open_retrieval", + "clarification", +) + +# Terminal operator -> interpretation family. Derived from the operator +# vocabulary of the frozen K132-K145 panels. +OPERATION_FAMILY = { + "bounded_tool": "registered_source_action", + "bounded_agentic_tool": "registered_source_action", + "compile_corpus_expert": "corpus_compile", + "native_relational_essay": "essay_artifact", + "clarify_essay_topic": "essay_artifact", + "clarify_essay_evidence": "essay_artifact", + "assert_modal_relation": "typed_modal", + "query_modal_relation": "typed_modal", + "compose_causal_graph": "causal_discourse", + "compose_causal_clause": "causal_discourse", + "query_causal_clause": "causal_discourse", + "query_persistent_causal_discourse": "causal_discourse", + "lexical_concept_definition": "lexical_definition", + "assert_graph_fact": "discourse_assertion", + "social_acknowledgement": "social", + "knowledge_explanation": "open_retrieval", +} + +# Panel expected-kind -> family, for gold turns that constrain the operator +# only negatively (operation_not) or structurally. +KIND_FAMILY = { + "public_essay": "essay_artifact", + "caller_essay": "essay_artifact", + "missing_topic": "essay_artifact", + "unknown_topic": "essay_artifact", + "modal_control": "typed_modal", + "modal_abstain": "typed_modal", + "modal_assert": "typed_modal", + "modal_query": "typed_modal", + "registered_tool": "registered_source_action", + "rapid_expert": "corpus_compile", +} + + +@dataclass(frozen=True) +class InterpretationCandidate: + family: str + resonance: float + feature_evidence: dict[str, float] + bid: float + + def public(self) -> dict[str, Any]: + return { + "family": self.family, + "resonance": round(self.resonance, 6), + "feature_evidence": { + name: round(value, 4) for name, value in self.feature_evidence.items() + }, + "bid": round(self.bid, 6), + } + + +@dataclass(frozen=True) +class ArbitrationRecord: + utterance_sha256: str + candidates: tuple[InterpretationCandidate, ...] + winner: str + runner_up: str + margin: float + clarification_floor_applied: bool + geometric_prototype_states: bool = True + + def public(self) -> dict[str, Any]: + return { + "utterance_sha256": self.utterance_sha256, + "candidates": [value.public() for value in self.candidates], + "winner": self.winner, + "runner_up": self.runner_up, + "margin": round(self.margin, 6), + "clarification_floor_applied": self.clarification_floor_applied, + "geometric_prototype_states": self.geometric_prototype_states, + "simultaneous_candidate_scoring": True, + } + + +@dataclass +class DiscourseSummary: + """Light typed discourse state the arbiter conditions bids on.""" + + causal_subjects: set[str] = field(default_factory=set) + modal_subjects: set[str] = field(default_factory=set) + graph_subjects: set[str] = field(default_factory=set) + registered_addresses: set[str] = field(default_factory=set) + + def observe(self, family: str, subject: str | None) -> None: + if not subject: + return + value = subject.casefold() + if family == "causal_discourse": + self.causal_subjects.add(value) + elif family == "typed_modal": + self.modal_subjects.add(value) + elif family == "discourse_assertion": + self.graph_subjects.add(value) + elif family == "registered_source_action": + self.registered_addresses.add(value) + + +def gold_family(expected: dict[str, Any]) -> str | None: + operation = str(expected.get("operation") or "") + if operation in OPERATION_FAMILY: + return OPERATION_FAMILY[operation] + kind = str(expected.get("kind") or "") + if kind in KIND_FAMILY: + return KIND_FAMILY[kind] + return None + + +__all__ = [ + "FAMILIES", "OPERATION_FAMILY", "KIND_FAMILY", + "InterpretationCandidate", "ArbitrationRecord", "DiscourseSummary", + "gold_family", +] diff --git a/root/wander-train/stage5k13_concurrent_specialist_pool.py b/root/wander-train/stage5k13_concurrent_specialist_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..31e335eb26bb11ff515340e652ad5be4eaa51126 --- /dev/null +++ b/root/wander-train/stage5k13_concurrent_specialist_pool.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Stage 5K13 concurrent general-domain HRR specialist pool. + +Sixteen small specialists inspect the same situated query concurrently. Each +cell contributes a typed signal packet and a deterministic HRR direction; none +can emit final prose. The three strongest cells form a query-local hybrid +specialist whose moving state is carried under the immutable 4:1 protected +root. Release is terminal-only. +""" + +from __future__ import annotations + +import hashlib +import re +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import Any, Mapping, Sequence + +import numpy as np + +from stage5k10_self_inverse_computation import digest +from train_stage2a_role_ladder import ( + HRR_DIM, + cosine, + normalize, + text_hrr, + unitary, +) + + +SCHEMA = "oracle-stage5k13-concurrent-specialist-pool-v1" +TOKEN_RE = re.compile(r"[a-z0-9]+", re.IGNORECASE) +NUMBER_RE = re.compile(r"-?\d+(?:\.\d+)?") + + +def tokens(text: str) -> tuple[str, ...]: + return tuple(TOKEN_RE.findall(str(text).casefold())) + + +def directional_text_state(text: str) -> np.ndarray: + """Order-aware plus order-tolerant native routing state. + + Signed word identities, position buckets, adjacent directional relations, + and character 3-5-gram shapes make a capability direction recoverable when + a paraphrase moves the same content to different token positions. The + sparse signed state avoids unrelated dense-vector correlation. Every + component is locally hashed; there is no learned embedding encoder. + """ + values = tokens(text)[:96] or ("empty",) + state = np.zeros(HRR_DIM, dtype=np.complex64) + + def add(feature: str, weight: float) -> None: + raw = hashlib.blake2b(feature.encode("utf-8"), digest_size=8).digest() + index = int.from_bytes(raw[:4], "little") % HRR_DIM + sign = 1.0 if raw[4] & 1 else -1.0 + if raw[5] & 1: + state[index] += np.complex64(1j * sign * weight) + else: + state[index] += np.complex64(sign * weight) + + previous: str | None = None + for index, token in enumerate(values): + add("word:" + token, 1.0) + add(f"position:{index % 4}:{token}", 0.16) + padded = "^" + token + "$" + for width in (3, 4, 5): + for index in range(max(0, len(padded) - width + 1)): + add("char:" + padded[index:index + width], 0.055) + if previous is not None: + add(f"next:{previous}:{token}", 0.35) + previous = token + return normalize(state) + + +def directional_similarity(a: np.ndarray, b: np.ndarray) -> float: + """Signed cosine for learned routing directions (opposition is not identity).""" + denominator = float(np.linalg.norm(a) * np.linalg.norm(b)) + if denominator <= 1e-12: + return 0.0 + return float(np.real(np.vdot(a, b)) / denominator) + + +@dataclass(frozen=True) +class SpecialistSpec: + specialist_id: str + competencies: tuple[str, ...] + cues: tuple[str, ...] + member_specialist_ids: tuple[str, ...] = () + + @property + def members(self) -> tuple[str, ...]: + return self.member_specialist_ids or (self.specialist_id,) + + def as_dict(self) -> dict[str, Any]: + return { + "specialist_id": self.specialist_id, + "competencies": list(self.competencies), + "cues": list(self.cues), + "member_specialist_ids": list(self.members), + } + + +@dataclass(frozen=True) +class SpecialistContribution: + specialist_id: str + activated_member_specialist_id: str + relevance: float + lexical_overlap: float + geometric_similarity: float + signals: tuple[tuple[str, Any], ...] + worker_name: str + started_ns: int + finished_ns: int + state: np.ndarray + + def as_dict(self) -> dict[str, Any]: + return { + "specialist_id": self.specialist_id, + "activated_member_specialist_id": self.activated_member_specialist_id, + "relevance": self.relevance, + "lexical_overlap": self.lexical_overlap, + "geometric_similarity": self.geometric_similarity, + "signals": [[key, value] for key, value in self.signals], + "worker_name": self.worker_name, + "started_ns": self.started_ns, + "finished_ns": self.finished_ns, + "state_sha256": hashlib.sha256( + np.ascontiguousarray(self.state).tobytes() + ).hexdigest(), + "final_prose_emitted": False, + } + + +@dataclass(frozen=True) +class SpecialistFusion: + query_sha256: str + contributions: tuple[SpecialistContribution, ...] + selected_specialist_ids: tuple[str, ...] + registered_specialist_count: int + concurrent_worker_count: int + protected_root_cosine: float + pre_malkuth_unbinds: int + malkuth_unbinds: int + fused_state: np.ndarray + protected_state: np.ndarray + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA, + "query_sha256": self.query_sha256, + "contributions": [value.as_dict() for value in self.contributions], + "selected_specialist_ids": list(self.selected_specialist_ids), + "selected_activated_member_specialist_ids": [ + value.activated_member_specialist_id + for value in self.contributions + if value.specialist_id in self.selected_specialist_ids + ], + "selected_specialist_count": len(self.selected_specialist_ids), + "registered_specialist_count": self.registered_specialist_count, + "concurrent_worker_count": self.concurrent_worker_count, + "protected_root_cosine": self.protected_root_cosine, + "pre_malkuth_unbinds": self.pre_malkuth_unbinds, + "malkuth_unbinds": self.malkuth_unbinds, + "malkuth_only_release": ( + self.pre_malkuth_unbinds == 0 and self.malkuth_unbinds == 1 + ), + "fused_state_sha256": hashlib.sha256( + np.ascontiguousarray(self.fused_state).tobytes() + ).hexdigest(), + "protected_state_sha256": hashlib.sha256( + np.ascontiguousarray(self.protected_state).tobytes() + ).hexdigest(), + "traditional_language_model": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "global_gradient_steps": 0, + } + + +DEFAULT_SPECIALISTS = ( + SpecialistSpec("linguistic", ("syntax", "questions", "roles"), ("what", "who", "where", "which", "describe", "write")), + SpecialistSpec("definition", ("definitions", "novel concepts"), ("define", "means", "is a", "in this conversation")), + SpecialistSpec("event", ("events", "thematic roles", "consequences"), ("gave", "transfer", "moved", "drove", "afterward", "caused")), + SpecialistSpec("causal", ("causal graphs", "counterfactuals"), ("because", "cause", "caused", "why", "so", "therefore")), + SpecialistSpec("mechanism", ("multi-hop mechanisms", "explanations"), ("chain", "fuse", "current", "motor", "mechanism", "why")), + SpecialistSpec("spatial", ("motion", "paths", "locations"), ("where", "across", "into", "from", "to", "moved", "orbit")), + SpecialistSpec("state", ("state transitions", "invariants"), ("changed", "increased", "decreased", "fixed", "stable", "became")), + SpecialistSpec("contrast", ("comparison", "difference", "unchanged"), ("contrast", "compare", "larger", "higher", "while", "difference")), + SpecialistSpec("numeric", ("arithmetic", "units", "verification"), ("compute", "times", "minus", "plus", "value", "temperature", "depth")), + SpecialistSpec("tool_router", ("bounded tools", "operation selection"), ("use", "calculator", "lookup", "find", "catalog", "observation")), + SpecialistSpec("dataset", ("schemas", "rows", "fields", "extrema"), ("population", "earthquake", "catalog", "country", "field", "deepest")), + SpecialistSpec("provenance", ("sources", "licenses", "authority"), ("who published", "provenance", "source", "license", "authority")), + SpecialistSpec("epistemic", ("abstention", "evidence boundaries"), ("prove", "did", "cause", "only", "without overstating", "unsupported")), + SpecialistSpec("corpus_expert", ("rapid corpus acquisition", "bounded facts"), ("supplied corpus", "using only", "world", "measured", "orbit")), + SpecialistSpec("sensor", ("streams", "trends", "stability"), ("sensor", "stream", "temperature", "pressure", "trend", "approximately")), + SpecialistSpec("synthesis", ("scientific synthesis", "evidence writing"), ("paragraph", "scientific", "synthesize", "explain", "evidence", "concise")), +) + + +class ConcurrentSpecialistPool: + def __init__( + self, + specs: Sequence[SpecialistSpec] = DEFAULT_SPECIALISTS, + *, + selected_count: int = 3, + maximum_workers: int = 4, + minimum_registered_count: int = 10, + cue_state_banks: Mapping[ + str, Sequence[tuple[str, np.ndarray]] + ] | None = None, + lexical_weight: float = 0.72, + geometric_weight: float = 0.28, + ) -> None: + self.specs = tuple(specs) + minimum_registered_count = int(minimum_registered_count) + if not 1 <= minimum_registered_count <= 20: + raise ValueError("minimum registered specialist count must be in 1-20") + if not minimum_registered_count <= len(self.specs) <= 20: + raise ValueError( + f"general pool requires {minimum_registered_count}-20 specialists" + ) + self.minimum_registered_count = minimum_registered_count + if len({value.specialist_id for value in self.specs}) != len(self.specs): + raise ValueError("specialist ids must be unique") + self.selected_count = max(2, min(3, int(selected_count))) + self.maximum_workers = max(2, min(int(maximum_workers), len(self.specs))) + if lexical_weight < 0.0 or geometric_weight < 0.0: + raise ValueError("specialist routing weights must be nonnegative") + weight_sum = float(lexical_weight + geometric_weight) + if weight_sum <= 0.0: + raise ValueError("at least one specialist routing weight must be positive") + self.lexical_weight = float(lexical_weight / weight_sum) + self.geometric_weight = float(geometric_weight / weight_sum) + supplied = dict(cue_state_banks or {}) + self._cue_state_banks: dict[ + str, tuple[tuple[str, np.ndarray], ...] + ] = {} + for value in self.specs: + if value.specialist_id in supplied: + bank = tuple( + (str(member), normalize(np.asarray(state, dtype=np.complex64))) + for member, state in supplied[value.specialist_id] + ) + if not bank: + raise ValueError("learned cue-state bank cannot be empty") + unknown = {member for member, _ in bank} - set(value.members) + if unknown: + raise ValueError( + f"cue-state bank contains unregistered members: {sorted(unknown)}" + ) + self._cue_state_banks[value.specialist_id] = bank + continue + state = normalize(sum( + (directional_text_state(cue) for cue in value.cues), + np.zeros(HRR_DIM, dtype=np.complex64), + )) + self._cue_state_banks[value.specialist_id] = tuple( + (member, state) for member in value.members + ) + + @staticmethod + def _signals( + specialist_id: str, + text: str, + context: Mapping[str, Any], + ) -> tuple[tuple[str, Any], ...]: + values = tokens(text) + word_set = set(values) + signals: dict[str, Any] = {} + if specialist_id == "linguistic": + signals["question"] = "?" in text or bool(word_set & {"what", "who", "where", "which", "why", "how"}) + signals["token_count"] = len(values) + elif specialist_id == "definition": + match = re.search(r"\ba\s+([a-z][a-z0-9_-]*)\s+is\s+a\s+", text.casefold()) + signals["defined_symbol"] = match.group(1) if match else None + elif specialist_id == "event": + signals["transfer_cue"] = bool(word_set & {"gave", "give", "transfer", "transfers"}) + signals["motion_cue"] = bool(word_set & {"moved", "move", "drove", "drive"}) + elif specialist_id in {"causal", "mechanism"}: + signals["causal_markers"] = sum( + text.casefold().count(value) for value in ("because", " so ", "why", "cause") + ) + elif specialist_id == "spatial": + signals["path_markers"] = [ + value for value in values if value in {"across", "into", "from", "to", "where"} + ] + elif specialist_id == "state": + signals["state_markers"] = [ + value for value in values if value in {"increased", "decreased", "fixed", "stable", "still"} + ] + elif specialist_id == "contrast": + signals["contrast_requested"] = bool(word_set & {"contrast", "compare", "while", "larger"}) + elif specialist_id == "numeric": + signals["numbers"] = NUMBER_RE.findall(text) + elif specialist_id == "tool_router": + signals["explicit_tool"] = next(( + value for value in ("calculator", "catalog", "observation") + if value in word_set + ), None) + elif specialist_id == "dataset": + signals["dataset_terms"] = sorted(word_set & {"population", "earthquake", "catalog", "canada", "mexico"}) + elif specialist_id == "provenance": + signals["provenance_requested"] = bool(word_set & {"published", "provenance", "source", "license"}) + elif specialist_id == "epistemic": + signals["causation_requested"] = "cause" in word_set or "caused" in word_set + signals["bounded_only"] = "only" in word_set + elif specialist_id == "corpus_expert": + corpus = context.get("training_corpus", ()) + signals["corpus_records"] = len(corpus) if isinstance(corpus, Sequence) else 0 + elif specialist_id == "sensor": + stream = context.get("sensor_stream", ()) + signals["sensor_records"] = len(stream) if isinstance(stream, Sequence) else 0 + elif specialist_id == "synthesis": + evidence = context.get("evidence", ()) + signals["evidence_records"] = len(evidence) if isinstance(evidence, Sequence) else 0 + return tuple(sorted(signals.items())) + + def _contribute( + self, + spec: SpecialistSpec, + text: str, + context: Mapping[str, Any], + query_state: np.ndarray, + barrier: threading.Barrier | None, + barrier_ids: frozenset[str], + ) -> SpecialistContribution: + started = time.perf_counter_ns() + if barrier is not None and spec.specialist_id in barrier_ids: + barrier.wait(timeout=5.0) + query_words = set(tokens(text)) + cue_words = set(tokens(" ".join(spec.cues))) + overlap = len(query_words & cue_words) / max(1, len(query_words | cue_words)) + bank = self._cue_state_banks[spec.specialist_id] + grouped: dict[str, list[tuple[float, np.ndarray]]] = {} + for member, member_state in bank: + grouped.setdefault(member, []).append(( + max(0.0, directional_similarity(query_state, member_state)), + member_state, + )) + member_scores = [] + for member, values in grouped.items(): + ranked = sorted(values, key=lambda value: -value[0]) + neighborhood = ranked[:min(5, len(ranked))] + geometric = ( + 0.35 * neighborhood[0][0] + + 0.65 * float(np.mean([value[0] for value in neighborhood])) + ) + activated_state = normalize(sum( + ( + np.float32(max(score, 1e-4)) * state + for score, state in neighborhood[:3] + ), + np.zeros(HRR_DIM, dtype=np.complex64), + )) + member_scores.append((member, geometric, activated_state)) + activated_member, geometric, activated_state = max( + member_scores, + key=lambda value: (value[1], value[0]), + ) + context_boost = 0.0 + if activated_member == "corpus_expert" and context.get("training_corpus"): + context_boost = 0.5 + elif activated_member == "sensor" and context.get("sensor_stream"): + context_boost = 0.5 + elif activated_member == "synthesis" and context.get("evidence"): + context_boost = 0.4 + relevance = min( + 1.0, + self.lexical_weight * overlap + + self.geometric_weight * geometric + + context_boost, + ) + signals = self._signals(activated_member, text, context) + signal_state = text_hrr( + activated_member + " " + " ".join( + f"{key} {value}" for key, value in signals + ) + ) + state = normalize( + np.float32(0.65) * activated_state + + np.float32(0.35) * signal_state + ) + return SpecialistContribution( + specialist_id=spec.specialist_id, + activated_member_specialist_id=activated_member, + relevance=float(relevance), + lexical_overlap=float(overlap), + geometric_similarity=float(geometric), + signals=signals, + worker_name=threading.current_thread().name, + started_ns=started, + finished_ns=time.perf_counter_ns(), + state=state, + ) + + def route( + self, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ) -> SpecialistFusion: + clean = " ".join(str(text).split()) + if not clean: + raise ValueError("specialist routing requires a nonempty query") + local_context = dict(context or {}) + query_state = directional_text_state(clean) + ablated = {str(value) for value in ablate_specialists} + barrier_specs = tuple( + value.specialist_id for value in self.specs + if value.specialist_id not in ablated + )[:self.maximum_workers] + barrier = ( + threading.Barrier(len(barrier_specs)) + if len(barrier_specs) >= 2 else None + ) + with ThreadPoolExecutor( + max_workers=self.maximum_workers, + thread_name_prefix="oracle-specialist", + ) as executor: + futures = [ + executor.submit( + self._contribute, + spec, clean, local_context, query_state, + barrier, frozenset(barrier_specs), + ) + for spec in self.specs if spec.specialist_id not in ablated + ] + contributions = tuple(value.result() for value in futures) + ordered = tuple(sorted( + contributions, + key=lambda value: (-value.relevance, value.specialist_id), + )) + selected = ordered[:self.selected_count] + moving = normalize(sum( + ( + np.float32(max(value.relevance, 1e-4)) * value.state + for value in selected + ), + np.zeros(HRR_DIM, dtype=np.complex64), + )) + root = unitary("stage5k13:protected-root:" + hashlib.sha256( + clean.encode("utf-8") + ).hexdigest()) + residual = moving - np.complex64(np.vdot(root, moving)) * root + protected = normalize(np.float32(4.0) * root + residual) + return SpecialistFusion( + query_sha256=hashlib.sha256(clean.encode("utf-8")).hexdigest(), + contributions=ordered, + selected_specialist_ids=tuple( + value.specialist_id for value in selected + ), + registered_specialist_count=len(self.specs), + concurrent_worker_count=len({ + value.worker_name for value in contributions + }), + protected_root_cosine=cosine(root, protected), + pre_malkuth_unbinds=0, + malkuth_unbinds=1, + fused_state=moving, + protected_state=protected, + ) + + def ablation_changes( + self, + text: str, + context: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + baseline = self.route(text, context) + ablated = self.route( + text, context, + ablate_specialists=baseline.selected_specialist_ids, + ) + fused_cosine = cosine(baseline.fused_state, ablated.fused_state) + return { + "baseline_selected": list(baseline.selected_specialist_ids), + "ablated_selected": list(ablated.selected_specialist_ids), + "selected_set_changed": ( + baseline.selected_specialist_ids != ablated.selected_specialist_ids + ), + "fused_state_cosine": fused_cosine, + "material_change": ( + baseline.selected_specialist_ids != ablated.selected_specialist_ids + and fused_cosine < 0.95 + ), + } + + +__all__ = [ + "ConcurrentSpecialistPool", + "DEFAULT_SPECIALISTS", + "directional_text_state", + "directional_similarity", + "SCHEMA", + "SpecialistContribution", + "SpecialistFusion", + "SpecialistSpec", +] diff --git a/root/wander-train/stage5k144_evidence_conditioned_sense_agent.py b/root/wander-train/stage5k144_evidence_conditioned_sense_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..d173ad0c7cc3956ecf7f346103582dd2e00f8c0a --- /dev/null +++ b/root/wander-train/stage5k144_evidence_conditioned_sense_agent.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""K144 evidence-conditioned dictionary sense selection. + +Live probes of frozen K141 showed public technical essays selecting +domain-incongruent dictionary senses: "heritable variation" resolved to the +navigation sense of variation (magnetic declination) and "consumer demand" +to the peremptory-request sense. The inherited K134 sense scorer already +rewards overlap between a sense's gloss vocabulary and the evidence rows — +but for public essays the definitions are computed before any evidence is +retrieved, so the scorer receives an empty context and degenerates to +sense-rank order. + +This successor supplies the missing ingredient rather than new scoring +machinery: before sense selection for a topic with no caller evidence, a +lightweight provisional retrieval over the topic's role families collects +real corpus vocabulary, and the unchanged inherited scorer disambiguates +against it. Caller-bounded behavior is unchanged (caller facts already +provide the context). No answer values, answer surfaces, language model, +transformer, pretrained encoder, autoregressive carrier, remote generator, +or gradient update is introduced. +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from contextvars import ContextVar +from typing import Any + +from stage5k15_native_mixed_agent import clean, digest +from stage5k81_information_weighted_source_expert import terms +from stage5k141_endpoint_joint_retrieval_agent import EndpointJointRetrievalAgent + + +SCHEMA = "oracle-stage5k144-evidence-conditioned-sense-agent-v1" +_K144_AUDIT: ContextVar[dict[str, Any] | None] = ContextVar( + "k144_evidence_conditioned_sense_audit", default=None, +) + + +class EvidenceConditionedSenseAgent(EndpointJointRetrievalAgent): + """Disambiguate dictionary senses against provisional corpus context.""" + + def __init__(self, *args: Any, k144_ablations: Sequence[str] = (), **kwargs: Any) -> None: + self._k144_ablations = frozenset(clean(value).casefold() for value in k144_ablations) + super().__init__(*args, **kwargs) + self._k144_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k141_lineage_sha256, + "opened_failure_source": "live_probe_domain_incongruent_sense_selection", + "operator": "provisional_corpus_context_for_inherited_sense_scorer", + }) + + def _provisional_context_rows(self, topic: str) -> tuple[dict[str, Any], ...]: + graph = self._topic_role_graph(topic) + families = {role: self._role_family(role) for role in graph.roles} + variants = sorted({ + value for family in families.values() for value in family + if len(value) > 2 and value.isalpha() + }) + if not variants: + return () + expression = " OR ".join(f'"{value}"' for value in variants) + rows = self.information_source_expert.connection.execute( + "SELECT fact, source_document FROM source_rows WHERE source_rows MATCH ? " + "ORDER BY bm25(source_rows) LIMIT 48", + (expression,), + ).fetchall() + scored: list[tuple[int, int, str]] = [] + for index, row in enumerate(rows): + fact = clean(row[0]).rstrip(".?!") + source = str(row[1]) + if not (source.startswith("openstax/") or source.startswith("gutenberg/")): + continue + if not 45 <= len(fact) <= 420 or fact.endswith("?"): + continue + fact_terms = self._expanded_terms(fact) + matched_roles = sum(1 for family in families.values() if fact_terms & family) + if matched_roles < 1: + continue + scored.append((-matched_roles, index, fact)) + scored.sort() + if not scored: + return () + # Keep only the most topical tier: rows matching the greatest number + # of distinct roles carry the topic's actual relational context, while + # single-role rows contribute off-topic vocabulary that can tip sense + # selection toward incidental readings. + best = scored[0][0] + tier = [row for row in scored if row[0] == best] + return tuple({"fact": fact} for _roles, _index, fact in tier[:12]) + + @staticmethod + def _sense_rank_value(sense: Any) -> int: + suffix = str(sense.sense).rsplit(".", 1)[-1] + return int(suffix) if suffix.isdigit() else 99 + + def _graph_pos_requirements(self, topic: str) -> dict[str, frozenset[str]]: + # The typed role graph is the authority on syntactic category: central + # endpoints are nominal arguments and predicates are verbal. The + # dictionary must agree with the construction, not with surface shape. + graph = self._topic_role_graph(topic) + required: dict[str, frozenset[str]] = {} + # Inflections only: the requirement binds the role surface itself, not + # its whole lemma family — a family term like "natural" (from the + # natural_selection lemma) must not inherit the endpoint's noun + # requirement. + for role in graph.endpoints: + for variant in self._inflections(role): + required[variant] = frozenset("n") + for role in graph.predicates: + for variant in self._inflections(role): + required[variant] = frozenset("v") + return required + + def _repick_sense( + self, + item: Mapping[str, Any], + required_pos: frozenset[str], + context_terms: set[str], + ) -> dict[str, Any]: + if str(item.get("part_of_speech")) in required_pos: + return dict(item) + # The catalog lookup short-circuits at the first variant with entries + # (e.g. "acts" resolves to the proper-noun book and hides act.v), so + # candidates are unioned across every inflection variant. + seen: set[str] = set() + candidates = [] + for variant in self._inflections(str(item["surface"])): + for sense in self.lexical_concepts.senses(variant): + if sense.part_of_speech in required_pos and sense.sense not in seen: + seen.add(sense.sense) + candidates.append(sense) + if not candidates: + return dict(item) + def overlap(sense: Any) -> int: + document = set(terms(" ".join(( + sense.definition, *sense.examples, *sense.hypernyms, *sense.lemmas, + )))) + return len(context_terms & document) + best = max(candidates, key=lambda sense: (overlap(sense), -self._sense_rank_value(sense))) + replaced = dict(item) + replaced.update({ + "concept": best.sense.rsplit(".", 2)[0].replace("_", " "), + "definition": clean(best.definition).rstrip(".?!"), + "sense": best.sense, + "part_of_speech": best.part_of_speech, + "lexname": best.lexname, + "lemmas": list(best.lemmas), + "hypernyms": list(best.hypernyms), + "selection": { + **dict(item.get("selection") or {}), + "selected": best.sense, + "graph_pos_repick": True, + "required_part_of_speech": sorted(required_pos), + "context_overlap": overlap(best), + }, + }) + return replaced + + def _topic_overlap(self, item: Mapping[str, Any], topic_terms: set[str]) -> int: + document = set(terms(" ".join(( + str(item.get("definition", "")), + " ".join(map(str, item.get("lemmas", ()))), + " ".join(map(str, item.get("hypernyms", ()))), + )))) + return len(topic_terms & document) + + def _concept_definitions( + self, topic: str, evidence_rows: Sequence[Mapping[str, Any]] = (), + ) -> tuple[dict[str, Any], ...]: + if "evidence_conditioned_senses" in self._k144_ablations: + return super()._concept_definitions(topic, evidence_rows) + base = super()._concept_definitions(topic, evidence_rows) + context_rows = tuple(evidence_rows) + provisional = False + if not context_rows: + context_rows = self._provisional_context_rows(topic) + provisional = bool(context_rows) + topic_terms = set(terms(topic)) + merged: list[dict[str, Any]] = [] + conditioned_by_surface: dict[str, Mapping[str, Any]] = {} + replacements = 0 + if provisional: + conditioned = super()._concept_definitions(topic, context_rows) + conditioned_by_surface = { + clean(str(item.get("surface", ""))).casefold(): item + for item in conditioned + } + for item in base: + surface = clean(str(item.get("surface", ""))).casefold() + candidate = conditioned_by_surface.get(surface) + # Consensus rule: the corpus-conditioned choice replaces the + # inherited choice only when the inherited winner was uninformed — + # selected by sense-rank default rather than by content overlap + # with the topic itself. An informed inherited choice (the + # development-blessed memory/identity senses, for example) is + # kept authoritative. + if ( + candidate is not None + and str(candidate.get("sense")) != str(item.get("sense")) + and self._topic_overlap(item, topic_terms) < 2 + ): + replaced = dict(candidate) + replaced["selection"] = { + **dict(candidate.get("selection") or {}), + "corpus_conditioned_replacement": True, + "inherited_sense": str(item.get("sense")), + } + merged.append(replaced) + replacements += 1 + else: + merged.append(dict(item)) + required = self._graph_pos_requirements(topic) + context_terms = topic_terms | { + value for row in context_rows for value in terms(str(row.get("fact", ""))) + } + repicked = [] + repick_count = 0 + for item in merged: + surface = clean(str(item.get("surface", ""))).casefold() + requirement = required.get(surface) + if requirement: + replaced = self._repick_sense(item, requirement, context_terms) + repick_count += replaced.get("selection", {}).get("graph_pos_repick") is True + repicked.append(replaced) + else: + repicked.append(dict(item)) + audit = _K144_AUDIT.get() + if audit is not None: + # The definitions path runs several times per turn (topic pass, + # endpoint-candidate passes, cached document pass); counters + # accumulate across passes so later benign passes cannot erase + # the record of an earlier replacement. + audit["activated"] = True + audit["provisional_context"] = bool(audit.get("provisional_context")) or provisional + audit["provisional_context_rows"] = max( + int(audit.get("provisional_context_rows", 0)), + len(context_rows) if provisional else 0, + ) + audit["corpus_conditioned_replacements"] = ( + int(audit.get("corpus_conditioned_replacements", 0)) + replacements + ) + audit["graph_pos_repicks"] = ( + int(audit.get("graph_pos_repicks", 0)) + repick_count + ) + audit["topic_sha256"] = digest(topic) + return tuple(repicked) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + audit: dict[str, Any] = {} + token = _K144_AUDIT.set(audit) + try: + output = super().answer(*args, **kwargs) + finally: + _K144_AUDIT.reset(token) + output["evidence_conditioned_senses"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k144_lineage_sha256, + "development_source": "opened_live_probe_sense_incongruence", + "prospective_claim": False, + "audit": audit, + "ablations": sorted(self._k144_ablations), + "inherited_scorer_unchanged": True, + "caller_context_source_unchanged": True, + "graph_pos_alignment_applies_to_all_essay_paths": True, + "answer_values_encoded": 0, + "answer_surfaces_in_training": 0, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "next_token_predictions": 0, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"].update({ + "evidence_conditioned_senses_model_calls": 0, + "evidence_conditioned_senses_gradient_steps": 0, + "evidence_conditioned_senses_next_token_predictions": 0, + }) + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["EvidenceConditionedSenseAgent", "SCHEMA"] diff --git a/root/wander-train/stage5k150_domain_scoped_retrieval_agent.py b/root/wander-train/stage5k150_domain_scoped_retrieval_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..ebf910b095157fe11613b0abfbe082936515ec9d --- /dev/null +++ b/root/wander-train/stage5k150_domain_scoped_retrieval_agent.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""K150 domain-scoped retrieval: the thirty-expert registry goes live. + +Every essay topic is routed through the K149 sparse domain profiles to its +two most resonant experts, and the two K-line-owned auxiliary retrieval +passes — the K144 provisional sense context and the K141 endpoint-joint +pass — are scoped to those experts' sources, with an automatic global +fallback when the scoped pass yields nothing. The routing decision and +scope outcome are recorded on every essay turn. + +This is precision work with a measured motive: the live probe that exposed +the sense-incongruence defect showed an off-domain computer-security row +entering the memory/identity sense context solely because it shared two +surface words with the topic. Domain scoping removes that class of +pollution at its entry points. The frozen main recall pass is deliberately +unchanged in this stage. No answer values, answer surfaces, language +model, transformer, pretrained encoder, autoregressive carrier, remote +generator, or gradient update is introduced. +""" +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping, Sequence +from contextvars import ContextVar +import gzip +import json +import math +from pathlib import Path +import re +from typing import Any + +from stage5k15_native_mixed_agent import clean, digest +from stage5k147_arbitrated_dispatch_agent import ArbitratedDispatchAgent + + +SCHEMA = "oracle-stage5k150-domain-scoped-retrieval-agent-v1" +_K150_AUDIT: ContextVar[dict[str, Any] | None] = ContextVar( + "k150_domain_scoped_retrieval_audit", default=None, +) + +_P = Path(__file__).resolve().parent.parent / "fable-content" / "oracle-phase3-2026-07-21" +REGISTRY_PATH = _P / "stage5k149_domain_expert_registry_v1.json" +PROFILES_PATH = _P / "stage5k149_domain_expert_registry_v1_profiles.json.gz" + +_TOKEN = re.compile(r"[a-z][a-z'-]{2,}") + + +class DomainScopedRetrievalAgent(ArbitratedDispatchAgent): + """Route essay topics to registry experts and scope auxiliary retrieval.""" + + def __init__(self, *args: Any, k150_ablations: Sequence[str] = (), **kwargs: Any) -> None: + self._k150_ablations = frozenset(clean(value).casefold() for value in k150_ablations) + super().__init__(*args, **kwargs) + registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) + with gzip.open(PROFILES_PATH, "rt", encoding="utf-8") as handle: + payload = json.load(handle) + self._k150_registry = registry + self._k150_profiles: dict[str, dict[str, float]] = payload["profiles"] + self._k150_document_frequency: dict[str, int] = payload["document_frequency"] + self._k150_domain_count: int = int(payload["domain_count"]) + self._k150_sources: dict[str, tuple[str, ...]] = { + name: tuple(spec["sources"]) for name, spec in registry["domains"].items() + } + self._k150_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k147_lineage_sha256, + "registry_sha256": digest(registry), + "operator": "sparse_profile_domain_routing_scoping_auxiliary_retrieval", + }) + + def _k150_route(self, text: str) -> dict[str, Any]: + tokens = [ + token for token in _TOKEN.findall(clean(text).casefold()) + ] + probe = Counter(tokens) + scores: list[tuple[float, str]] = [] + for name, profile in self._k150_profiles.items(): + score = sum( + count + * math.log(1.0 + self._k150_domain_count / self._k150_document_frequency.get(token, 1)) + * profile.get(token, 0.0) + for token, count in probe.items() + ) + scores.append((score, name)) + scores.sort(reverse=True) + return { + "top1": scores[0][1], "top2": scores[1][1], + "top1_score": round(scores[0][0], 6), "top2_score": round(scores[1][0], 6), + "margin": round(scores[0][0] - scores[1][0], 6), + } + + def _k150_scope(self, topic: str) -> tuple[str, ...]: + if "domain_scoped_retrieval" in self._k150_ablations: + return () + routing = self._k150_route(topic) + sources = ( + *self._k150_sources[routing["top1"]], + *self._k150_sources[routing["top2"]], + ) + audit = _K150_AUDIT.get() + if audit is not None: + audit.setdefault("routings", []).append({ + "topic_sha256": digest(topic), **routing, + "scoped_source_count": len(sources), + }) + audit["activated"] = True + return sources + + @staticmethod + def _k150_filter( + rows: Sequence[Mapping[str, Any]], sources: tuple[str, ...], key: str, + ) -> tuple[dict[str, Any], ...]: + allowed = set(sources) + return tuple(dict(row) for row in rows if str(row.get(key)) in allowed) + + def _provisional_context_rows(self, topic: str) -> tuple[dict[str, Any], ...]: + # The inherited pass keeps only the most topical tier but carries no + # domain notion; rows are re-fetched with their source labels so the + # registry scope can exclude off-domain lexical coincidences. + sources = self._k150_scope(topic) + if not sources: + return super()._provisional_context_rows(topic) + graph = self._topic_role_graph(topic) + families = {role: self._role_family(role) for role in graph.roles} + variants = sorted({ + value for family in families.values() for value in family + if len(value) > 2 and value.isalpha() + }) + if not variants: + return super()._provisional_context_rows(topic) + expression = " OR ".join(f'"{value}"' for value in variants) + placeholders = ",".join("?" for _ in sources) + rows = self.information_source_expert.connection.execute( + "SELECT fact, source_document FROM source_rows WHERE source_rows MATCH ? " + f"AND source_document IN ({placeholders}) " + "ORDER BY bm25(source_rows) LIMIT 48", + (expression, *sources), + ).fetchall() + scored: list[tuple[int, int, str]] = [] + for index, row in enumerate(rows): + fact = clean(row[0]).rstrip(".?!") + if not 45 <= len(fact) <= 420 or fact.endswith("?"): + continue + fact_terms = self._expanded_terms(fact) + matched = sum(1 for family in families.values() if fact_terms & family) + if matched < 1: + continue + scored.append((-matched, index, fact)) + scored.sort() + audit = _K150_AUDIT.get() + if not scored: + if audit is not None: + audit["provisional_scope_fallback_to_global"] = True + return super()._provisional_context_rows(topic) + best = scored[0][0] + tier = [row for row in scored if row[0] == best] + if audit is not None: + audit["provisional_rows_scoped"] = len(tier[:12]) + return tuple({"fact": fact} for _matched, _index, fact in tier[:12]) + + def _joint_candidates( + self, left: str, right: str, roles: Sequence[str], + ) -> tuple[dict[str, Any], ...]: + candidates = super()._joint_candidates(left, right, roles) + sources = self._k150_scope(" ".join(roles)) + if not sources: + return candidates + scoped = self._k150_filter(candidates, sources, "source_document") + audit = _K150_AUDIT.get() + if scoped: + if audit is not None: + audit["joint_candidates_scoped"] = len(scoped) + return scoped + if audit is not None and candidates: + audit["joint_scope_fallback_to_global"] = True + return candidates + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + audit: dict[str, Any] = {} + token = _K150_AUDIT.set(audit) + try: + output = super().answer(*args, **kwargs) + finally: + _K150_AUDIT.reset(token) + output["domain_scoped_retrieval"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k150_lineage_sha256, + "development_source": "stage5k149_domain_expert_registry_v1", + "prospective_claim": False, + "audit": audit, + "ablations": sorted(self._k150_ablations), + "registered_domains": len(self._k150_sources), + "main_recall_pass_unchanged": True, + "global_fallback_preserved": True, + "answer_values_encoded": 0, + "answer_surfaces_in_training": 0, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "next_token_predictions": 0, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"].update({ + "domain_scoped_retrieval_model_calls": 0, + "domain_scoped_retrieval_gradient_steps": 0, + "domain_scoped_retrieval_next_token_predictions": 0, + }) + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["DomainScopedRetrievalAgent", "SCHEMA"] diff --git a/root/wander-train/stage5k152_typed_goal_carrier_removal_agent.py b/root/wander-train/stage5k152_typed_goal_carrier_removal_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..65c429e087dbe2e0e3d9894bccbfc16a616c31db --- /dev/null +++ b/root/wander-train/stage5k152_typed_goal_carrier_removal_agent.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""K152 typed-goal carrier removal. + +K131 recognizes a registered-source action construction, extracts every +typed relation it needs — source domain, extremum direction, metric, +requested projection fields — and then serializes those relations back into +a canonical English sentence solely so the inherited surface compiler can +recover the typed plan it already had. That canonical-English internal +carrier is architecturally redundant: language should occur at the ear and +at Malkuth, not as the private API between interpretation and control. + +This successor compiles the ToolAction plan directly from the typed +construction and proves exact equivalence: the typed compiler emits +byte-identical ToolAction tuples to the carrier path for every surface in +the construction family, and execution, audit, and provenance are unchanged. +The carrier path is retained only as the ablation control. No answer +values, answer surfaces, language model, transformer, pretrained encoder, +autoregressive carrier, remote generator, or gradient update is introduced. +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from contextvars import ContextVar +from typing import Any + +from stage5k15_native_mixed_agent import HandlerResult, clean, digest +from stage5k86_native_goal_tool_controller import ToolAction +from stage5k131_registered_source_action_ingress_agent import _K131_AUDIT +from stage5k150_domain_scoped_retrieval_agent import DomainScopedRetrievalAgent + + +SCHEMA = "oracle-stage5k152-typed-goal-carrier-removal-agent-v1" +_K152_AUDIT: ContextVar[dict[str, Any] | None] = ContextVar( + "k152_typed_goal_carrier_removal_audit", default=None, +) + + +class TypedGoalCarrierRemovalAgent(DomainScopedRetrievalAgent): + """Compile registered-source plans from typed relations, not English.""" + + def __init__(self, *args: Any, k152_ablations: Sequence[str] = (), **kwargs: Any) -> None: + self._k152_ablations = frozenset(clean(value).casefold() for value in k152_ablations) + super().__init__(*args, **kwargs) + self._k152_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k150_lineage_sha256, + "removed_carrier": "k131_canonical_english_reparse", + "operator": "typed_construction_to_tool_action_compilation", + }) + + def _compile_construction_goal( + self, construction: Mapping[str, Any], fields: tuple[str, ...], + ) -> tuple[ToolAction, ...]: + """Byte-identical ToolAction output to the carrier path, typed input.""" + domain = ( + "earthquake database" + if construction["domain"] == "earthquake" + else "population database" + ) + extremum = str(construction["direction"]) + metric = "magnitude" if domain == "earthquake database" else "value" + select = ToolAction( + "action_1", + extremum, + domain, + f"From the {domain}, find the {extremum} {metric}.", + produces_address="observation:selected_row", + required_fields=(metric, "row_key"), + ) + if domain != "earthquake database" or not fields: + return (select,) + project = ToolAction( + "action_2", + "lookup", + domain, + "From the earthquake database, retrieve ${observation:selected_row}; " + + "provide " + " and ".join(fields) + ".", + depends_on=(select.action_id,), + consumes_address="observation:selected_row", + produces_address="observation:selected_fields", + required_fields=fields, + ) + return (select, project) + + def _carrier_plan( + self, construction: Mapping[str, Any], text: str, + ) -> tuple[tuple[ToolAction, ...], tuple[str, ...]]: + """The K131 carrier path, kept verbatim as the equivalence control.""" + domain = ( + "earthquake database" + if construction["domain"] == "earthquake" + else "population database" + ) + metric = "magnitude" if domain == "earthquake database" else "value" + fields = self._requested_fields(text, domain) + projection = " and ".join(fields) + carrier = ( + f"From the {domain}, find the {construction['direction']} {metric}; " + "then provide the selected record" + + (f" and {projection}." if projection else ".") + ) + return self._compile_goal(carrier), fields + + def _project_registered_source_action( + self, text: str, context: Mapping[str, Any], ingress: str, + ) -> HandlerResult | None: + if "typed_goal_direct" in self._k152_ablations: + return super()._project_registered_source_action(text, context, ingress) + construction = self._registered_source_action(text) + if ( + construction is None + or "registered_source_action_precedence" in self._k131_ablations + ): + return None + session_id = clean(context.get("k102_session_id", "")) + if not session_id: + return None + domain = ( + "earthquake database" + if construction["domain"] == "earthquake" + else "population database" + ) + fields = self._requested_fields(text, domain) + plan = self._compile_construction_goal(construction, fields) + result = ( + self._execute_goal(session_id, text, plan) + if plan else self._bounded_tool_graph(session_id, text) + ) + typed_goal_sha256 = digest([ + { + "action_id": action.action_id, "operation": action.operation, + "domain": action.domain, "depends_on": list(action.depends_on), + "required_fields": list(action.required_fields), + } + for action in plan + ]) + audit = _K131_AUDIT.get() + if audit is not None: + audit.update({ + **construction, + "ingress": ingress, + "selected_operation": result.operation, + "selected_status": result.status, + "tool_graph_precedes_open_public_text": True, + "surface_rewritten": False, + "construction_projected_to_typed_goal": bool(plan), + "typed_goal_action_count": len(plan), + "typed_goal_sha256": typed_goal_sha256, + "canonical_english_carrier_used": False, + }) + k152 = _K152_AUDIT.get() + if k152 is not None: + k152.update({ + "typed_goal_direct": True, + "typed_goal_sha256": typed_goal_sha256, + "typed_goal_action_count": len(plan), + }) + return HandlerResult( + result.operation, result.status, result.arguments, result.result, + result.clauses, result.evidence, + ("recognize_whole_registered_source_action_construction", + "compile_typed_goal_without_english_carrier", + "bind_tool_precedence_before_open_public_text", + *result.computation_trace), + tool_audit={ + **dict(result.tool_audit or {}), + "k131_whole_construction_precedence": True, + "k131_surface_rewritten": False, + "k131_construction_projected_to_typed_goal": bool(plan), + "k152_typed_goal_direct": True, + }, + memory_audit=result.memory_audit, + ) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + audit: dict[str, Any] = {} + token = _K152_AUDIT.set(audit) + try: + output = super().answer(*args, **kwargs) + finally: + _K152_AUDIT.reset(token) + output["typed_goal_carrier_removal"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k152_lineage_sha256, + "development_source": "k137_handoff_direct_carrier_removal_item", + "prospective_claim": False, + "audit": audit, + "ablations": sorted(self._k152_ablations), + "carrier_seam_removed": "k131_registered_source_action", + "canonical_english_reparse_removed_globally": False, + "answer_values_encoded": 0, + "answer_surfaces_in_training": 0, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "next_token_predictions": 0, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"].update({ + "typed_goal_carrier_removal_model_calls": 0, + "typed_goal_carrier_removal_gradient_steps": 0, + "typed_goal_carrier_removal_next_token_predictions": 0, + }) + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["TypedGoalCarrierRemovalAgent", "SCHEMA"] diff --git a/root/wander-train/stage5k156_wiki_augmented_retrieval_agent.py b/root/wander-train/stage5k156_wiki_augmented_retrieval_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..455376a5c14ce05735c1f39eba5b84cab9151e28 --- /dev/null +++ b/root/wander-train/stage5k156_wiki_augmented_retrieval_agent.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""K156 Wikipedia-augmented fallback retrieval. + +The curated 105-source index remains the primary evidence tier. When it has +nothing for a topic — no joint endpoint rows, no selected evidence, no +provisional sense context — the one-million-article Wikipedia abstract index +(K155, schema-identical, CC-BY-SA provenance per row) answers as the +fallback tier. Curated behavior is byte-unchanged wherever the curated +corpus has support; the augmentation activates only on emptiness, and every +activation is recorded. + +No answer values, answer surfaces, language model, transformer, pretrained +encoder, autoregressive carrier, remote generator, or gradient update is +introduced. +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from contextvars import ContextVar +import hashlib +from pathlib import Path +import sqlite3 +from typing import Any + +from stage5k15_native_mixed_agent import clean, digest +from stage5k152_typed_goal_carrier_removal_agent import TypedGoalCarrierRemovalAgent + + +SCHEMA = "oracle-stage5k156-wiki-augmented-retrieval-agent-v1" +_K156_AUDIT: ContextVar[dict[str, Any] | None] = ContextVar( + "k156_wiki_augmented_retrieval_audit", default=None, +) + +WIKI_INDEX = ( + Path(__file__).resolve().parent.parent + / "fable-content" / "oracle-phase3-2026-07-21" + / "stage5k155_wiki_abstract_index_v1.sqlite3" +) + + +class WikiAugmentedRetrievalAgent(TypedGoalCarrierRemovalAgent): + """Curated-first retrieval with a recorded Wikipedia fallback tier.""" + + def __init__(self, *args: Any, k156_ablations: Sequence[str] = (), **kwargs: Any) -> None: + self._k156_ablations = frozenset(clean(value).casefold() for value in k156_ablations) + super().__init__(*args, **kwargs) + self._k156_wiki = sqlite3.connect(f"file:{WIKI_INDEX}?mode=ro", uri=True) + self._k156_wiki_sha256 = hashlib.sha256(WIKI_INDEX.read_bytes()).hexdigest() + self._k156_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k152_lineage_sha256, + "wiki_index_sha256": self._k156_wiki_sha256, + "operator": "curated_first_wiki_fallback_on_empty", + }) + + def _k156_active(self) -> bool: + return "wiki_fallback_retrieval" not in self._k156_ablations + + def _k156_record(self, event: str, count: int) -> None: + audit = _K156_AUDIT.get() + if audit is not None: + audit.setdefault("fallbacks", []).append({"event": event, "rows": count}) + audit["activated"] = True + + def _k156_wiki_rows(self, expression: str, limit: int) -> list[tuple]: + return self._k156_wiki.execute( + "SELECT text_sha256, fact, source_document, source_document_sha256, " + "domain_id, license_note, source_split, bm25(source_rows) " + "FROM source_rows WHERE source_rows MATCH ? " + "ORDER BY bm25(source_rows) LIMIT ?", + (expression, limit), + ).fetchall() + + def _k156_quality(self, fact: str) -> bool: + return ( + 45 <= len(fact) <= 420 + and not fact.endswith("?") + and not self._ROW_REJECT.search(fact) + and "â" not in fact and "�" not in fact + ) + + def _wiki_joint_candidates( + self, left: str, right: str, roles: Sequence[str], + ) -> tuple[dict[str, Any], ...]: + expression = self._joint_expression(left, right) + if not expression: + return () + families = {role: self._role_family(role) for role in roles} + ranked: list[tuple[float, float, str, dict[str, Any]]] = [] + for row in self._k156_wiki_rows(expression, 64): + fact = clean(row[1]).rstrip(".?!") + if not self._k156_quality(fact): + continue + fact_terms = self._expanded_terms(fact) + matched = [role for role, family in families.items() if fact_terms & family] + if left not in matched or right not in matched: + continue + score = 14.0 * len(matched) + 2.0 * (75 <= len(fact) <= 300) + ranked.append((score, float(row[7]), str(row[0]), { + "text_sha256": str(row[0]), "fact": fact, + "source_document": str(row[2]), "source_document_sha256": str(row[3]), + "domain_id": str(row[4]), "license_note": str(row[5]), + "source_split": str(row[6]), "matched_topic_terms": list(matched), + "matched_query_variants": sorted( + fact_terms & families[left] | fact_terms & families[right] + ), + "caller_bounded": False, + })) + ranked.sort(key=lambda item: (-item[0], item[1], item[2])) + return tuple(row for _score, _bm25, _sha, row in ranked) + + def _joint_candidates( + self, left: str, right: str, roles: Sequence[str], + ) -> tuple[dict[str, Any], ...]: + candidates = super()._joint_candidates(left, right, roles) + if candidates or not self._k156_active(): + return candidates + wiki = self._wiki_joint_candidates(left, right, roles) + if wiki: + self._k156_record("joint_candidates_from_wiki", len(wiki)) + return wiki + + def _indexed_evidence( + self, topic: str, definitions: Sequence[Mapping[str, Any]], + ) -> tuple[dict[str, Any], ...]: + selected = super()._indexed_evidence(topic, definitions) + if selected or not self._k156_active(): + return selected + graph = self._topic_role_graph(topic) + if len(graph.endpoints) < 2: + return selected + wiki = self._wiki_joint_candidates( + graph.endpoints[0], graph.endpoints[-1], graph.roles, + ) + if not wiki: + return selected + chosen: list[dict[str, Any]] = [] + seen: set[str] = set() + for row in wiki: + if row["text_sha256"] not in seen: + seen.add(row["text_sha256"]) + chosen.append(dict(row)) + if len(chosen) == 3: + break + self._k156_record("indexed_evidence_from_wiki", len(chosen)) + return tuple(chosen) + + def _provisional_context_rows(self, topic: str) -> tuple[dict[str, Any], ...]: + rows = super()._provisional_context_rows(topic) + if rows or not self._k156_active(): + return rows + graph = self._topic_role_graph(topic) + families = {role: self._role_family(role) for role in graph.roles} + variants = sorted({ + value for family in families.values() for value in family + if len(value) > 2 and value.isalpha() + }) + if not variants: + return rows + expression = " OR ".join(f'"{value}"' for value in variants) + scored: list[tuple[int, int, str]] = [] + for index, row in enumerate(self._k156_wiki_rows(expression, 48)): + fact = clean(row[1]).rstrip(".?!") + if not self._k156_quality(fact): + continue + fact_terms = self._expanded_terms(fact) + matched = sum(1 for family in families.values() if fact_terms & family) + if matched < 1: + continue + scored.append((-matched, index, fact)) + scored.sort() + if not scored: + return rows + best = scored[0][0] + tier = [item for item in scored if item[0] == best] + self._k156_record("provisional_context_from_wiki", len(tier[:12])) + return tuple({"fact": fact} for _matched, _index, fact in tier[:12]) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + audit: dict[str, Any] = {} + token = _K156_AUDIT.set(audit) + try: + output = super().answer(*args, **kwargs) + finally: + _K156_AUDIT.reset(token) + output["wiki_augmented_retrieval"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k156_lineage_sha256, + "development_source": "stage5k155_wiki_abstract_index_v1", + "prospective_claim": False, + "wiki_index_sha256": self._k156_wiki_sha256, + "wiki_rows_indexed": 2495302, + "license": "CC-BY-SA 4.0 (Wikipedia abstract snapshot)", + "audit": audit, + "ablations": sorted(self._k156_ablations), + "curated_tier_primary": True, + "fallback_only_on_empty": True, + "answer_values_encoded": 0, + "answer_surfaces_in_training": 0, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "next_token_predictions": 0, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"].update({ + "wiki_augmented_retrieval_model_calls": 0, + "wiki_augmented_retrieval_gradient_steps": 0, + "wiki_augmented_retrieval_next_token_predictions": 0, + }) + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["WikiAugmentedRetrievalAgent", "SCHEMA", "WIKI_INDEX"] diff --git a/root/wander-train/stage5k171_relational_utterance_compiler.py b/root/wander-train/stage5k171_relational_utterance_compiler.py new file mode 100644 index 0000000000000000000000000000000000000000..58b37daf81daa614d235ff57d5ffc4b4166a376f --- /dev/null +++ b/root/wander-train/stage5k171_relational_utterance_compiler.py @@ -0,0 +1,525 @@ +#!/usr/bin/env python3 +"""K171 bounded-English compiler into a persistent HRR role/filler interior. + +The compiler does not create one ontology node per sentence. A small stable +operator repertoire is kept separate from query-local fillers and scoped +variables. Resulting expressions are recursive values in the existing K1 +workspace and are stabilized by its dual-rail Tree spine. The final utterance +wrapper follows a valid route through all ten nodes and unbinds only at +Malkuth. +""" +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import re +from typing import Any, Mapping, Sequence + +from stage5k1_generative_relational_workspace import ( + CanonicalRelationalSpace, + EphemeralRelationalWorkspace, + OperatorDefinition, + Provenance, + RelationalConstraint, + RelationalHRRCodec, + RelationalState, + RelationalValue, + RelationalVariable, + default_canonical_space, + substitute_expression, +) +from train_stage2a_role_ladder import cosine + + +SCHEMA = "oracle-stage5k171-relational-utterance-compiler-v1" +WORD = re.compile(r"[A-Za-z0-9][A-Za-z0-9'_-]*") +ALL_NODE_ROUTE = ( + "keter", "chokmah", "binah", "geburah", "tiferet", + "chesed", "netzach", "hod", "yesod", "malkuth", +) +TREE_NODES = ALL_NODE_ROUTE +TREE_PATH_REPERTOIRE = 22 +NULL_FILLER = "" + +_TRANSFER = re.compile( + r"^(.+?)\s+(?:transfers?|gives?|hands?|passes?|donates?)\s+(.+?)\s+to\s+(.+)$", + re.IGNORECASE, +) +_MOVE = re.compile( + r"^(.+?)\s+(?:moves?|travels?|goes?)\s+from\s+(.+?)\s+to\s+(.+)$", + re.IGNORECASE, +) +_CHANGE = re.compile( + r"^(.+?)\s+changes?\s+(.+?)\s+from\s+(.+?)\s+to\s+(.+)$", + re.IGNORECASE, +) +_CONTAIN = re.compile(r"^(.+?)\s+(?:contains?|holds?|encloses?)\s+(.+)$", re.IGNORECASE) +_CAUSE = re.compile(r"^(.+?)\s+(?:causes?|produces?|results?\s+in)\s+(.+)$", re.IGNORECASE) +_CONDITION = re.compile(r"^if\s+(.+?),\s*(?:then\s+)?(.+)$", re.IGNORECASE) +_UNTIL = re.compile(r"^(.+?)\s+until\s+(.+)$", re.IGNORECASE) +_COUNTERFACTUAL = re.compile( + r"^(?:counterfactually|suppose|imagine),?\s+(?:that\s+)?(.+)$", + re.IGNORECASE, +) +_NEGATION = re.compile(r"\b(?:does\s+not|do\s+not|did\s+not|never)\b", re.IGNORECASE) +_RELATION_HINTS = frozenset({ + "repel", "repair", "own", "paint", "guard", "carry", "touch", + "observe", "support", "follow", "open", "close", "enter", "leave", + "heat", "cool", "measure", "signal", "orbit", "cross", "join", +}) + + +def _sha(value: str) -> str: + return hashlib.sha256(str(value).encode("utf-8")).hexdigest() + + +def _surface(value: str) -> str: + text = " ".join(str(value).strip().strip(" .?!,:;\"'").split()) + return re.sub(r"^(?:a|an|the)\s+", "", text, flags=re.IGNORECASE) + + +def _lemma(value: str) -> str: + word = value.casefold() + irregular = {"gave": "give", "given": "give", "went": "go", "held": "hold"} + if word in irregular: + return irregular[word] + if word.endswith("ies") and len(word) > 4: + return word[:-3] + "y" + if word.endswith("es") and len(word) > 4: + return word[:-2] + if word.endswith("s") and not word.endswith(("ss", "us")) and len(word) > 3: + return word[:-1] + if word.endswith("ed") and len(word) > 4: + stem = word[:-2] + return stem[:-1] if len(stem) > 2 and stem[-1] == stem[-2] else stem + return word + + +def canonical_space() -> CanonicalRelationalSpace: + space = default_canonical_space() + space.register(OperatorDefinition( + "relation", ("predicate", "agent", "patient"), + ("value", "value", "value"), "state", ALL_NODE_ROUTE, + laws=("role_order_is_invariant", "fillers_are_query_local"), + source="k171_compressed_relation_family", + )) + space.register(OperatorDefinition( + "utterance_state", ("speech_act", "content"), + ("value", "state"), "state", ALL_NODE_ROUTE, + laws=("surface_is_not_the_reasoning_state", "malkuth_only_egress"), + source="k171_relational_interlayer", + )) + return space + + +@dataclass(frozen=True) +class RoleRecovery: + state_id: str + operator_id: str + role: str + filler_kind: str + filler_type: str + filler_identity_sha256: str + filler_surface_sha256: str | None + recovery_cosine: float + cleanup_margin: float + cleanup_correct: bool + variable_scope_sha256: str | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "state_id": self.state_id, "operator_id": self.operator_id, + "role": self.role, "filler_kind": self.filler_kind, + "filler_type": self.filler_type, + "filler_identity_sha256": self.filler_identity_sha256, + "filler_surface_sha256": self.filler_surface_sha256, + "recovery_cosine": self.recovery_cosine, + "cleanup_margin": self.cleanup_margin, + "cleanup_correct": self.cleanup_correct, + "variable_scope_sha256": self.variable_scope_sha256, + } + + +@dataclass(frozen=True) +class CompiledUtterance: + supported: bool + reason: str + speech_act: str + root_state: RelationalState | None + frame_states: tuple[RelationalState, ...] + role_recoveries: tuple[RoleRecovery, ...] + variable_ids: tuple[str, ...] + source_sha256: str + world_id_sha256: str | None = None + counterfactual_world: bool = False + + def public(self) -> dict[str, Any]: + root = self.root_state + evaluations = [value.evaluation for value in self.frame_states] + if root is not None: + evaluations.append(root.evaluation) + return { + "schema_version": SCHEMA, + "supported": self.supported, "reason": self.reason, + "speech_act": self.speech_act, + "source_sha256": self.source_sha256, + "world_id_sha256": self.world_id_sha256, + "counterfactual_world": self.counterfactual_world, + "root_state_id": None if root is None else root.state_id, + "root_expression_sha256": None if root is None else root.expression.structural_sha256, + "root_operator": None if root is None else root.expression.operator_id, + "expression_depth": None if root is None else root.expression.depth, + "frame_count": len(self.frame_states), + "frame_operators": [value.expression.operator_id for value in self.frame_states], + "role_recoveries": [value.as_dict() for value in self.role_recoveries], + "role_cleanup_accuracy": ( + sum(value.cleanup_correct for value in self.role_recoveries) + / max(1, len(self.role_recoveries)) + ), + "minimum_role_cleanup_margin": min( + (value.cleanup_margin for value in self.role_recoveries), default=0.0, + ), + "scoped_variables": len(self.variable_ids), + "variable_identity_sha256s": [_sha(value) for value in self.variable_ids], + "tree_nodes_available": len(TREE_NODES), + "tree_paths_available": TREE_PATH_REPERTOIRE, + "root_route": [] if root is None else list(root.evaluation.route), + "all_ten_nodes_traversed": bool( + root is not None and tuple(root.evaluation.route) == ALL_NODE_ROUTE + ), + "active_root_path_count": 0 if root is None else len(root.evaluation.route) - 1, + "protected_root_cosine_min": min( + (value.protected_root_cosine for value in evaluations), default=0.0, + ), + "inverse_reconstruction_exact": bool( + evaluations and all(value.inverse_reconstruction_exact for value in evaluations) + ), + "pre_malkuth_unbinds": sum(value.pre_malkuth_unbinds for value in evaluations), + "malkuth_unbinds": sum(value.malkuth_unbinds for value in evaluations), + "malkuth_only_unbinding": bool( + evaluations + and sum(value.pre_malkuth_unbinds for value in evaluations) == 0 + and all(value.malkuth_unbinds == 1 for value in evaluations) + ), + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "next_token_predictions": 0, + "model_calls": 0, + "gradient_steps": 0, + } + + +class RelationalUtteranceCompiler: + """Compile a bounded relational grammar into reusable operator shapes.""" + + def __init__(self) -> None: + self._sessions: dict[str, EphemeralRelationalWorkspace] = {} + + def workspace(self, session_id: str) -> EphemeralRelationalWorkspace: + key = str(session_id) + value = self._sessions.get(key) + if value is None: + value = EphemeralRelationalWorkspace(f"k171:{key}", canonical_space()) + self._sessions[key] = value + return value + + @staticmethod + def _relation_index(words: Sequence[str]) -> int | None: + for index, value in enumerate(words[1:-1], start=1): + lemma = _lemma(value) + if lemma in _RELATION_HINTS: + return index + for index, value in enumerate(words[1:-1], start=1): + lower = value.casefold() + if lower.endswith(("ed", "ing")) or ( + lower.endswith("s") and not lower.endswith(("ss", "us")) + ): + return index + return None + + @staticmethod + def _variable_key(surface: str) -> str | None: + value = _surface(surface).casefold() + if value in {"someone", "whoever", "person", "that person", "same person"}: + return "person" + if value in {"object", "that object", "same object", "it"}: + return "object" + if value in {"something", "thing", "that thing"}: + return "thing" + return None + + def _value( + self, workspace: EphemeralRelationalWorkspace, surface: str, value_type: str, + variables: dict[str, RelationalVariable], scope: str, + world_id: str | None, + ) -> RelationalValue: + clean_surface = _surface(surface) + key = self._variable_key(clean_surface) + if key is None: + return workspace.symbol(clean_surface, value_type, world_id=world_id) + variable = variables.get(key) + if variable is None: + variable = workspace.new_variable(key, value_type, scope_id=scope) + variables[key] = variable + return RelationalValue.variable(variable) + + @staticmethod + def _constraint(scope: str, text: str) -> tuple[RelationalConstraint, ...]: + polarity = "negative" if _NEGATION.search(text) else "positive" + return (RelationalConstraint("clause_scope", (scope,), scope, polarity),) + + def _compose( + self, workspace: EphemeralRelationalWorkspace, operator: str, + values: Sequence[RelationalValue | RelationalState], source: str, + constraints: Sequence[RelationalConstraint], + world_id: str | None, + ) -> RelationalState: + return workspace.compose( + operator, values, constraints=constraints, ttl_turns=128, + world_id=world_id, + provenance=Provenance.local(source), + ) + + def _clause( + self, workspace: EphemeralRelationalWorkspace, text: str, + variables: dict[str, RelationalVariable], scope: str, + world_id: str | None, + ) -> RelationalState | None: + value = _surface(text) + constraints = self._constraint(scope, value) + match = _TRANSFER.match(value) + if match: + return self._compose(workspace, "transfer", ( + self._value(workspace, match.group(1), "entity", variables, scope, world_id), + self._value(workspace, match.group(2), "entity", variables, scope, world_id), + self._value(workspace, match.group(3), "entity", variables, scope, world_id), + ), f"{scope}:transfer", constraints, world_id) + match = _MOVE.match(value) + if match: + return self._compose(workspace, "move", ( + self._value(workspace, match.group(1), "entity", variables, scope, world_id), + self._value(workspace, match.group(2), "place", variables, scope, world_id), + self._value(workspace, match.group(3), "place", variables, scope, world_id), + ), f"{scope}:move", constraints, world_id) + match = _CHANGE.match(value) + if match: + return self._compose(workspace, "change", ( + self._value(workspace, match.group(1), "entity", variables, scope, world_id), + self._value(workspace, match.group(2), "quality", variables, scope, world_id), + self._value(workspace, match.group(3), "value", variables, scope, world_id), + self._value(workspace, match.group(4), "value", variables, scope, world_id), + ), f"{scope}:change", constraints, world_id) + match = _CONTAIN.match(value) + if match: + return self._compose(workspace, "contain", ( + self._value(workspace, match.group(1), "entity", variables, scope, world_id), + self._value(workspace, match.group(2), "entity", variables, scope, world_id), + ), f"{scope}:contain", constraints, world_id) + match = _CAUSE.match(value) + if match: + left = self._clause(workspace, match.group(1), variables, scope + ":cause", world_id) + right = self._clause(workspace, match.group(2), variables, scope + ":result", world_id) + if left is not None and right is not None: + return self._compose( + workspace, "cause", (left, right), f"{scope}:cause", constraints, world_id, + ) + + words = WORD.findall(value) + index = self._relation_index(words) + if index is None: + return None + subject = " ".join(words[:index]) + predicate = _lemma(words[index]) + patient = " ".join(words[index + 1:]) + if not subject or not patient: + return None + if _NEGATION.search(value): + patient = _NEGATION.sub("", patient).strip() + return self._compose(workspace, "relation", ( + workspace.symbol(predicate, "relation", world_id=world_id), + self._value(workspace, subject, "entity", variables, scope, world_id), + self._value(workspace, patient, "entity", variables, scope, world_id), + ), f"{scope}:relation", constraints, world_id) + + @staticmethod + def _recoveries(states: Sequence[RelationalState]) -> tuple[RoleRecovery, ...]: + rows: list[RoleRecovery] = [] + for state in states: + expression = state.expression + candidates = [RelationalHRRCodec.encode_value(value) for value in expression.arguments] + for index, (role, value) in enumerate(zip(expression.roles, expression.arguments)): + recovered = RelationalHRRCodec.recover_argument(state.vector, role) + scores = [cosine(recovered, candidate) for candidate in candidates] + own = scores[index] + rivals = [score for other, score in enumerate(scores) if other != index] + runner = max(rivals, default=0.0) + variable_scope = None + if value.kind == "variable": + variable_scope = value.identity.split(":", 1)[-1] + rows.append(RoleRecovery( + state_id=state.state_id, operator_id=expression.operator_id, + role=role, filler_kind=value.kind, filler_type=value.value_type, + filler_identity_sha256=_sha(value.identity), + filler_surface_sha256=None if value.surface is None else _sha(value.surface.casefold()), + recovery_cosine=own, cleanup_margin=own - runner, + cleanup_correct=scores.index(max(scores)) == index, + variable_scope_sha256=None if variable_scope is None else _sha(variable_scope), + )) + return tuple(rows) + + def compile(self, session_id: str, text: str, speech_act: str = "proposition") -> CompiledUtterance: + source = " ".join(str(text).strip().split()) + source_sha = _sha(source) + workspace = self.workspace(session_id) + variables: dict[str, RelationalVariable] = {} + scope = f"scope:k171:{source_sha[:20]}" + parse_source = source.strip().rstrip(".?!") + counterfactual = _COUNTERFACTUAL.match(parse_source) + world_id = None + if counterfactual: + parse_source = counterfactual.group(1).strip() + world_id = workspace.branch_world( + f"counterfactual:{source_sha[:16]}", + ) + frames: list[RelationalState] = [] + content: RelationalState | None = None + condition = _CONDITION.match(parse_source) + if condition: + antecedent = self._clause( + workspace, condition.group(1), variables, scope + ":if", world_id, + ) + consequence_text = condition.group(2) + until = _UNTIL.match(consequence_text) + if until: + consequence = self._clause( + workspace, until.group(1), variables, scope + ":then", world_id, + ) + termination = self._clause( + workspace, until.group(2), variables, scope + ":until", world_id, + ) + if consequence is not None and termination is not None: + frames.extend((consequence, termination)) + consequence = self._compose( + workspace, "until", (consequence, termination), + f"{scope}:until", self._constraint(scope + ":until", consequence_text), + world_id, + ) + frames.append(consequence) + else: + consequence = self._clause( + workspace, consequence_text, variables, scope + ":then", world_id, + ) + if antecedent is not None and consequence is not None: + frames.insert(0, antecedent) + if not until: + frames.append(consequence) + content = self._compose( + workspace, "condition", (antecedent, consequence), + f"{scope}:condition", self._constraint(scope, source), world_id, + ) + frames.append(content) + else: + content = self._clause(workspace, parse_source, variables, scope, world_id) + if content is not None: + frames.append(content) + if content is None: + workspace.advance_turn(1) + return CompiledUtterance( + False, "bounded_relational_parse_absent", speech_act, None, (), (), (), + source_sha, None if world_id is None else _sha(world_id), bool(world_id), + ) + root = self._compose(workspace, "utterance_state", ( + workspace.symbol(speech_act, "relation", world_id=world_id), content, + ), f"{scope}:utterance", self._constraint(scope, source), world_id) + all_states = (*frames, root) + recoveries = self._recoveries(all_states) + workspace.advance_turn(1) + return CompiledUtterance( + True, "compiled_to_query_local_role_filler_state", speech_act, + root, tuple(frames), recoveries, + tuple(value.variable_id for value in variables.values()), source_sha, + _sha(root.world_id), bool(world_id), + ) + + @staticmethod + def _variables(expression: Any) -> tuple[RelationalValue, ...]: + rows: dict[str, RelationalValue] = {} + + def visit(value: Any) -> None: + for argument in value.arguments: + if argument.kind == "variable": + rows.setdefault(argument.identity, argument) + if argument.expression is not None: + visit(argument.expression) + + visit(expression) + return tuple(rows[key] for key in sorted(rows)) + + def substitute( + self, session_id: str, compiled: CompiledUtterance, + substitutions: Mapping[str, str], + ) -> CompiledUtterance: + """Capture-safe variable substitution without reparsing or retraining.""" + if not compiled.supported or compiled.root_state is None: + raise ValueError("substitution requires a compiled relational state") + workspace = self.workspace(session_id) + root_expression = compiled.root_state.expression + content_value = root_expression.arguments[1] + if content_value.expression is None: + raise ValueError("compiled utterance has no relational content") + variables = self._variables(content_value.expression) + supplied = {str(key).casefold(): str(value) for key, value in substitutions.items()} + replacements: dict[str, RelationalValue] = {} + for variable in variables: + key = str(variable.surface or "").casefold() + if key in supplied: + replacements[variable.identity] = workspace.symbol( + supplied[key], variable.value_type, + world_id=compiled.root_state.world_id, + ) + if not replacements: + raise ValueError("no compiled variable matches the supplied substitutions") + substituted = substitute_expression(content_value.expression, replacements) + before = {value.state.state_id for value in workspace.records} + content = workspace.store_expression( + substituted, world_id=compiled.root_state.world_id, ttl_turns=128, + provenance=Provenance.local( + f"k171:substitution:{compiled.source_sha256[:20]}", + parent_state_ids=(compiled.root_state.state_id,), + ), + ) + frames = tuple( + value.state for value in workspace.records + if value.state.state_id not in before + ) + source_sha = _sha( + compiled.source_sha256 + ":" + ":".join( + f"{key.casefold()}={value.casefold()}" + for key, value in sorted(substitutions.items()) + ) + ) + scope = f"scope:k171:substitute:{source_sha[:20]}" + root = self._compose(workspace, "utterance_state", ( + workspace.symbol( + "instantiated proposition", "relation", + world_id=compiled.root_state.world_id, + ), + content, + ), f"{scope}:utterance", self._constraint(scope, "substitution"), + compiled.root_state.world_id) + recoveries = self._recoveries((*frames, root)) + remaining = self._variables(root.expression) + workspace.advance_turn(1) + return CompiledUtterance( + True, "capture_safe_variable_substitution", "instantiated_proposition", + root, frames, recoveries, tuple(value.identity for value in remaining), + source_sha, _sha(root.world_id), compiled.counterfactual_world, + ) + + +__all__ = [ + "ALL_NODE_ROUTE", "CompiledUtterance", "RelationalUtteranceCompiler", + "RoleRecovery", "SCHEMA", "canonical_space", +] diff --git a/root/wander-train/stage5k20_compositional_construction_agent.py b/root/wander-train/stage5k20_compositional_construction_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..936e69a7a21c44ee4da4a0f9f62d036e8269a86c --- /dev/null +++ b/root/wander-train/stage5k20_compositional_construction_agent.py @@ -0,0 +1,761 @@ +#!/usr/bin/env python3 +"""K20 native agent: compositional construction algebra over K18. + +K20 does not add a language model. It adds the missing finite compiler layer +between English surfaces and the relational operators already executed by the +native runtime. Alternants such as the prepositional dative, double-object +dative, and passive normalize to one TRANSFER program; lexical aspect/state +alternants normalize to one STATE_CONTRAST program; connector order normalizes +to one CAUSAL_CHAIN program; and bounded arithmetic constructions normalize to +an exact Decimal expression. The resulting typed roles are still authorized +by the trained meta-fold specialist basis, persisted through the HRR substrate, +and realized only by the whole-construction inverse at Malkuth. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import ( + HandlerResult, + SemanticClause, + clean, + digest, + tokens, +) +from stage5k18_goal_constrained_native_agent import GoalConstrainedNativeAgent +from stage5i2_open_schema_conversational_transducer import display_number + + +SCHEMA = "oracle-stage5k20-compositional-construction-agent-v1" +CONSTRUCTION_SCHEMA = "oracle-stage5k20-construction-algebra-v1" + + +@dataclass(frozen=True) +class ConstructionFrame: + """A surface-independent relational program selected before execution.""" + + family: str + operator: str + surface_pattern: str + roles: tuple[tuple[str, str], ...] = () + canonical_form: str | None = None + + def as_dict(self) -> dict[str, Any]: + body = { + "schema_version": CONSTRUCTION_SCHEMA, + "family": self.family, + "operator": self.operator, + "surface_pattern": self.surface_pattern, + "roles": [[key, value] for key, value in self.roles], + "canonical_form": self.canonical_form, + "typed_operator_selected": self.operator != "UNRESOLVED", + "surface_words_are_not_answer_labels": True, + "traditional_language_model": False, + "pretrained_embedding_encoder": False, + "next_token_prediction": False, + } + body["frame_sha256"] = digest(body) + return body + + +def _strip_determiner(value: str) -> str: + return re.sub(r"^(?:a|an|the)\s+", "", clean(value), flags=re.IGNORECASE) + + +def _lemma(surface: str) -> str: + return { + "gave": "give", + "given": "give", + "handed": "hand", + "transferred": "transfer", + "moves": "move", + "moved": "move", + "carries": "carry", + "carried": "carry", + "drives": "drive", + "drove": "drive", + "rose": "increase", + "increased": "increase", + "grew": "increase", + "fell": "decrease", + "decreased": "decrease", + "declined": "decrease", + "changed": "change", + }.get(surface.casefold(), surface.casefold()) + + +class CompositionalConstructionNativeAgent(GoalConstrainedNativeAgent): + """K18 with a typed, paraphrase-invariant surface-to-program compiler.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + kwargs.setdefault("agent_id", "oracle-stage5k20-compositional-construction-agent") + super().__init__(*args, **kwargs) + self._last_tool_normalization: dict[str, Any] | None = None + + def _retrieve( + self, + session_id: str, + query_roles: Mapping[str, str], + turn_id: str, + *, + top_k: int = 8, + ) -> tuple[list[dict[str, str]], dict[str, Any]]: + """Make approximate HRR retrieval exact at the dialogue boundary. + + Geometry proposes handles; authorization after Malkuth still enforces + the symbolic session root. This prevents a nearby HRR root from + becoming cross-dialogue anaphora merely because it ranked in top-k. + """ + released, audit = super()._retrieve( + session_id, query_roles, turn_id, top_k=top_k, + ) + root = self._session_root(session_id) + scoped = [value for value in released if value.get("root") == root] + revised = dict(audit) + revised["geometric_authorized_releases"] = int(audit.get("authorized_releases", 0)) + revised["scope_rejected_releases"] = len(released) - len(scoped) + revised["authorized_releases"] = len(scoped) + revised["exact_session_root_enforced_after_malkuth"] = True + return scoped, revised + + @classmethod + def _parse_transfer(cls, text: str) -> tuple[ConstructionFrame, tuple[str, str, str]] | None: + surface = clean(text) + verb = r"(?:gives?|gave|hands?|handed|transfers?|transferred)" + + # Passive must be tested first: otherwise its embedded "handed to" + # substring can be misread as a malformed active dative. + match = re.fullmatch( + r"(?:the\s+)?(.+?)\s+(?:was|is)\s+(?:given|handed|transferred)\s+to\s+" + r"(?:the\s+)?([A-Za-z][A-Za-z0-9_-]*)\s+by\s+" + r"(?:the\s+)?([A-Za-z][A-Za-z0-9_-]*)[.!?]*", + surface, + re.IGNORECASE, + ) + if match: + item, recipient, actor = (clean(match.group(index)) for index in (1, 2, 3)) + pattern = "passive_prepositional_dative" + else: + # Active prepositional dative: AGENT gave THEME to RECIPIENT. + match = re.fullmatch( + rf"([A-Za-z][A-Za-z0-9_-]*)\s+{verb}\s+(?:the\s+)?(.+?)\s+to\s+([A-Za-z][A-Za-z0-9_-]*)[.!?]*", + surface, + re.IGNORECASE, + ) + pattern = "active_prepositional_dative" + if match: + actor, item, recipient = (clean(match.group(index)) for index in (1, 2, 3)) + else: + # Active double-object dative: AGENT gave RECIPIENT THEME. + match = re.fullmatch( + rf"([A-Za-z][A-Za-z0-9_-]*)\s+{verb}\s+([A-Za-z][A-Za-z0-9_-]*)\s+(?:the\s+)?(.+?)[.!?]*", + surface, + re.IGNORECASE, + ) + pattern = "active_double_object_dative" + if not match: + return None + actor, recipient, item = (clean(match.group(index)) for index in (1, 2, 3)) + item = _strip_determiner(item) + values = (actor, item, recipient) + return ConstructionFrame( + "event_construction", + "TRANSFER", + pattern, + (("agent", actor.casefold()), ("theme", item.casefold()), ("recipient", recipient.casefold())), + f"TRANSFER(agent={actor.casefold()},theme={item.casefold()},recipient={recipient.casefold()})", + ), values + + @classmethod + def _transfer(cls, text: str) -> tuple[str, str, str] | None: + parsed = cls._parse_transfer(text) + return parsed[1] if parsed else None + + @staticmethod + def _parse_state_contrast(text: str) -> tuple[ConstructionFrame, tuple[str, str, str, str]] | None: + match = re.search( + r"\b([A-Za-z][A-Za-z0-9_-]*)\s+" + r"(rose|increased|grew|fell|decreased|declined|changed)\s+while\s+" + r"([A-Za-z][A-Za-z0-9_-]*)\s+" + r"(held|stayed|remained)\s+(fixed|stable|unchanged|constant)\b", + clean(text), + re.IGNORECASE, + ) + if not match: + return None + changed = clean(match.group(1)).casefold() + transition = _lemma(match.group(2)) + invariant = clean(match.group(3)).casefold() + invariant_surface = clean(match.group(5)).casefold() + invariant_state = "fixed" if invariant_surface in {"constant", "unchanged"} else invariant_surface + frame = ConstructionFrame( + "state_construction", + "STATE_CONTRAST", + "change_while_invariant", + ( + ("changed_entity", changed), + ("transition", transition), + ("invariant_entity", invariant), + ("invariant_state", invariant_state), + ), + f"STATE_CONTRAST(changed={changed}:{transition},invariant={invariant}:{invariant_state})", + ) + return frame, (changed, transition, invariant, invariant_state) + + @staticmethod + def _parse_causal_chain(text: str) -> tuple[ConstructionFrame, tuple[str, str, str]] | None: + surface = clean(text) + because_after = re.search( + r"\b(?:the\s+)?(.+?)\s+because\s+(?:the\s+)?(.+?)\s+after\s+(?:the\s+)?(.+?)[.!?]*$", + surface, + re.IGNORECASE, + ) + if because_after: + effect, intermediate, cause = ( + clean(because_after.group(index)).casefold() for index in (1, 2, 3) + ) + pattern = "effect_because_intermediate_after_cause" + else: + so_chain = re.search( + r"\b(?:the\s+)?(.+?)\s*,?\s+so\s+(?:the\s+)?(.+?)\s+and\s+(?:the\s+)?(.+?)[.!?]*$", + surface, + re.IGNORECASE, + ) + if not so_chain: + return None + cause, intermediate, effect = ( + clean(so_chain.group(index)).casefold() for index in (1, 2, 3) + ) + pattern = "cause_so_intermediate_and_effect" + frame = ConstructionFrame( + "causal_construction", + "CAUSAL_CHAIN", + pattern, + (("cause", cause), ("intermediate", intermediate), ("effect", effect)), + f"CAUSE({cause}->{intermediate}->{effect})", + ) + return frame, (cause, intermediate, effect) + + @classmethod + def _bound_arithmetic(cls, text: str) -> tuple[float, str, str] | None: + normalized = clean(text).casefold().replace(",", "").replace("(", " ").replace(")", " ") + + twice = re.search( + r"\btwice\s+(-?\d+(?:\.\d+)?)" + r"(?:\s+(plus|minus)\s+(-?\d+(?:\.\d+)?))?", + normalized, + ) + if twice: + left = Decimal(twice.group(1)) + value = Decimal(2) * left + expression = f"(2*{cls._decimal_surface(left)})" + if twice.group(2): + adjustment = Decimal(twice.group(3)) + symbol = "+" if twice.group(2) == "plus" else "-" + value = value + adjustment if symbol == "+" else value - adjustment + expression += symbol + cls._decimal_surface(adjustment) + return float(value), cls._decimal_surface(value), expression + + division = re.search( + r"(?:\bdivide\s+(-?\d+(?:\.\d+)?)\s+by\s+(-?\d+(?:\.\d+)?)\b|" + r"\b(-?\d+(?:\.\d+)?)\s+divided\s+by\s+(-?\d+(?:\.\d+)?)\b)", + normalized, + ) + if division: + left_raw = division.group(1) or division.group(3) + right_raw = division.group(2) or division.group(4) + left, right = Decimal(left_raw), Decimal(right_raw) + if right == 0: + return None + try: + value = left / right + except InvalidOperation: + return None + expression = f"{cls._decimal_surface(left)}/{cls._decimal_surface(right)}" + return float(value), cls._decimal_surface(value), expression + return super()._bound_arithmetic(text) + + @staticmethod + def _routing_goal(text: str, context: Mapping[str, Any]) -> tuple[tuple[str, ...], ...]: + words = set(tokens(text)) + lower = clean(text).casefold() + if context.get("sensor_stream"): + return (("sensor", "state"),) + if context.get("training_corpus"): + return (("corpus_expert",), ("linguistic", "synthesis")) + if context.get("evidence"): + return (("synthesis", "epistemic"),) + if ( + words & {"calculate", "calculator", "compute", "multiply", "multiplied", "times", "plus", "minus", "subtract", "add", "twice", "divide", "divided"} + and len(re.findall(r"-?\d+(?:\.\d+)?", lower)) >= 2 + ): + return (("numeric", "tool_router"),) + if "in this dialogue" in lower or "in this conversation" in lower or re.search(r"\bwhat\s+(?:is|does)\b", lower): + return (("definition", "linguistic"),) + data_words = { + "population", "populations", "earthquake", "earthquakes", "catalog", "database", + "observation", "gdp", "license", "provenance", "publisher", "source", + } + if words & data_words or context.get("_opaque_prior_tool_context"): + if words & {"cause", "caused", "causal", "damage"}: + return (("dataset",), ("causal", "epistemic")) + if words & {"license", "provenance", "publisher", "source"}: + return (("provenance",),) + if words & {"compare", "differ", "count", "many"}: + return (("dataset",), ("numeric", "tool_router")) + return (("dataset",),) + if words & {"gave", "give", "gives", "given", "handed", "transfer", "transferred", "holds", "afterward"}: + return (("event",), ("linguistic", "causal")) + if words & {"drove", "pushed", "pulled", "carried", "moved", "motion", "across", "through", "where"}: + return (("spatial", "event"),) + if words & {"contrast", "fixed", "stable", "unchanged", "constant", "held", "rose", "fell", "increased", "decreased", "changed"}: + return (("state", "contrast"),) + if words & {"why", "because", "cause", "caused", "so", "after", "mechanism"}: + return (("causal", "mechanism"),) + return (("linguistic", "epistemic"),) + + @staticmethod + def _event_capability_family(text: str) -> tuple[str, ...]: + words = set(tokens(text)) + if words & { + "rose", "fell", "grew", "declined", "constant", "held", + "fixed", "stable", "unchanged", "increased", "decreased", "changed", + }: + return ("state", "contrast", "linguistic") + return GoalConstrainedNativeAgent._event_capability_family(text) + + @staticmethod + def _definition_action(meaning: str) -> tuple[str, str, str | None, str | None] | None: + match = re.search( + r"\b(?:that|which)\s+([a-z][a-z0-9_-]*)\s+(?:the\s+)?(.+?)" + r"(?:\s+(into|to|through|across|from|with)\s+(?:a|an|the)?\s*([a-z][a-z0-9_-]*))?$", + clean(meaning), + re.IGNORECASE, + ) + if not match: + return None + verb = _lemma(match.group(1)) + theme = _strip_determiner(match.group(2)) + preposition = clean(match.group(3)).casefold() if match.group(3) else None + oblique = clean(match.group(4)).casefold() if match.group(4) else None + return verb, theme, preposition, oblique + + def _handle_definition(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + # Interrogative arithmetic ("what is twice ...") is not a lexical + # definition merely because it shares the "what is" surface. + if self._bound_arithmetic(text) is not None: + return None + action_query = re.search( + r"\bwhat\s+does\s+(?:a|an|the)?\s*([a-z][a-z0-9_-]*)\s+([a-z][a-z0-9_-]*)", + clean(text), + re.IGNORECASE, + ) + if not action_query: + return super()._handle_definition(session_id, text, turn_id) + + referent = action_query.group(1).casefold() + requested_verb = _lemma(action_query.group(2)) + query = {"situation": "definition"} + if referent not in {"it", "this", "that"}: + query["field"] = referent + payloads, memory_audit = self._retrieve(session_id, query, turn_id, top_k=16) + definitions = [value for value in payloads if value.get("situation") == "definition"] + if referent not in {"it", "this", "that"}: + definitions = [value for value in definitions if value.get("field") == referent] + match = definitions[0] if definitions else None + if match: + action = self._definition_action(match.get("result", "")) + if action and action[0] == requested_verb: + verb, theme, preposition, oblique = action + qualifiers = ((preposition, oblique),) if preposition and oblique else () + return HandlerResult( + "definition_role_query", + "answered", + (("term", match["field"]), ("requested_relation", requested_verb)), + (("theme", theme), ("oblique", oblique)), + (SemanticClause(f"A {match['field']}", verb, object_value=theme, qualifiers=qualifiers),), + (("memory_snapshot", memory_audit["snapshot_digest"]),), + ( + "resolve_typed_discourse_referent", + "recover_definition_relation", + "release_requested_semantic_role", + ), + memory_audit=memory_audit, + ) + return self._abstention( + "definition_role_query", + f"what {referent} {requested_verb}s", + "definition_relation_not_in_session_memory", + memory_audit=memory_audit, + ) + + def _handle_event(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + state = self._parse_state_contrast(text) + if state: + _frame, (changed, transition, invariant, invariant_state) = state + memory_id = self._bind_memory( + session_id, + turn_id, + { + "situation": "state_contrast", + "source": changed, + "transition": transition, + "field": invariant, + "result": invariant_state, + }, + source="k20_conversation_state_contrast", + ) + return HandlerResult( + "record_state_contrast", + "answered", + (("changed", changed), ("invariant", invariant)), + (("transition", transition), ("invariant_state", invariant_state), ("memory_id", memory_id)), + ( + SemanticClause(changed, transition, tense="past"), + SemanticClause(invariant, "remain", complement=invariant_state, tense="past"), + ), + (("source", "conversation_state_contrast"),), + ( + "normalize_lexical_state_alternants", + "bind_state_transition", + "bind_invariant", + "record_contrast", + ), + ) + + words = set(tokens(text)) + if ( + words & {"which", "what"} + and words & {"changed", "rose", "fell", "increased", "decreased"} + and words & {"fixed", "stable", "constant", "unchanged", "stayed"} + ): + payloads, memory_audit = self._retrieve( + session_id, {"situation": "state_contrast"}, turn_id, top_k=16, + ) + match = next((value for value in payloads if value.get("situation") == "state_contrast"), None) + if match: + return HandlerResult( + "state_contrast", + "answered", + (), + ( + ("changed", match["source"]), + ("transition", match["transition"]), + ("invariant", match["field"]), + ("invariant_state", match["result"]), + ), + ( + SemanticClause(match["source"], match["transition"], tense="past"), + SemanticClause(match["field"], "remain", complement=match["result"], tense="past"), + ), + (("memory_snapshot", memory_audit["snapshot_digest"]),), + ( + "retrieve_state_contrast", + "release_transition_and_invariant", + "construct_contrast_answer", + ), + memory_audit=memory_audit, + ) + + causal = self._parse_causal_chain(text) + if causal: + _frame, (cause, intermediate, effect) = causal + memory_id = self._bind_memory( + session_id, + turn_id, + { + "situation": "causal_chain", + "source": cause, + "field": intermediate, + "result": effect, + "transition": "cause", + }, + source="k20_conversation_causal_chain", + ) + return HandlerResult( + "record_causal_chain", + "answered", + (("cause", cause), ("intermediate", intermediate), ("effect", effect)), + (("memory_id", memory_id),), + ( + self._surface_proposition_clause(cause), + self._surface_proposition_clause(intermediate), + self._surface_proposition_clause(effect), + ), + (("source", "conversation_causal_chain"),), + ( + "normalize_causal_connector_order", + "bind_ordered_causal_edges", + "record_two_hop_mechanism", + ), + ) + + entity = self._location_query(text) + if entity: + payloads, memory_audit = self._retrieve( + session_id, + {"situation": "caused_motion", "field": entity}, + turn_id, + top_k=16, + ) + match = next(( + value for value in payloads + if value.get("situation") == "caused_motion" and value.get("field") == entity + ), None) + if match: + destination = match["result"] + return HandlerResult( + "motion_consequence", + "answered", + (("entity", entity),), + (("location", destination),), + (SemanticClause(entity, "be", complement=f"in {destination}"),), + (("memory_snapshot", memory_audit["snapshot_digest"]),), + ( + "retrieve_caused_motion_handle", + "project_path_as_terminal_location", + "construct_location_clause", + ), + memory_audit=memory_audit, + ) + return super()._handle_event(session_id, text, turn_id) + + def _handle_corpus( + self, + session_id: str, + text: str, + context: Mapping[str, Any], + turn_id: str, + ) -> HandlerResult | None: + corpus = context.get("training_corpus") + if not ( + isinstance(corpus, Sequence) + and not isinstance(corpus, (str, bytes)) + and corpus + and all(isinstance(value, Mapping) for value in corpus) + ): + return super()._handle_corpus(session_id, text, context, turn_id) + + compiled: list[dict[str, str]] = [] + memory_ids: list[str] = [] + for index, item in enumerate(corpus): + entity = clean(item.get("entity", item.get("subject", f"record {index + 1}"))) + relation = clean(item.get("relation", item.get("predicate", "state"))).casefold() + value = clean(item.get("value", item.get("object", item.get("result", "")))) + source = clean(item.get("source", f"case-local-record-{index + 1}")) + if not value: + continue + record = {"entity": entity, "relation": relation, "value": value, "source": source} + compiled.append(record) + memory_ids.append(self._bind_memory( + session_id, + f"{turn_id}-corpus-{index}", + { + "situation": "case_local_expert", + "source": entity.casefold(), + "field": relation, + "result": value, + "warrant": source, + }, + bank="semantic", + source="k20_case_local_corpus", + )) + if not compiled: + return self._abstention("compile_corpus_expert", "the supplied corpus", "no_compilable_records") + + lower = clean(text).casefold() + query_words = set(tokens(text)) + mentioned_entities = { + row["entity"].casefold() for row in compiled + if row["entity"].casefold() in lower + } + candidates = [ + row for row in compiled + if not mentioned_entities or row["entity"].casefold() in mentioned_entities + ] + relevant = [ + row for row in candidates + if query_words & set(tokens(row["relation"].replace("_", " "))) + ] or candidates + relevant = relevant[:4] + clauses = tuple( + SemanticClause( + f"{row['entity']} {row['relation'].replace('_', ' ')}", + "be", + complement=row["value"], + ) + for row in relevant + ) + return HandlerResult( + "compile_corpus_expert", + "answered", + (("records", len(compiled)),), + (("released_records", len(clauses)), ("memory_ids", memory_ids)), + clauses, + tuple((row["source"], digest(row)) for row in relevant), + ( + "compile_case_local_relational_folds", + "normalize_schema_relation_surface", + "route_entity_then_relation", + "release_bounded_facts", + ), + tool_audit={ + "tool": "native_case_local_expert_compiler", + "training_records": len(compiled), + "global_gradient_steps": 0, + "automatic_canonical_promotions": 0, + }, + ) + + def _normalize_tool_question(self, text: str) -> str: + self._last_tool_normalization = None + if self.tool_backend is None or not hasattr(self.tool_backend, "experts"): + return text + words = set(tokens(text)) + lower = clean(text).casefold() + if not (words & {"compare", "versus", "differ"} or "by how much" in lower): + return text + + for expert in getattr(self.tool_backend, "experts", {}).values(): + positioned: list[tuple[int, Mapping[str, str]]] = [] + for row in expert.rows: + positions: list[int] = [] + for field, value in row.items(): + candidates = expert.candidate_values.get(field, set()) + surface = clean(value).casefold() + if len(candidates) > 1 and len(surface) >= 3 and surface in lower: + positions.append(lower.index(surface)) + if positions: + positioned.append((min(positions), row)) + positioned.sort(key=lambda value: value[0]) + matched = [row for _position, row in positioned] + if len(matched) != 2: + continue + keys = ["|".join(clean(row[field]) for field in expert.schema.primary_key) for row in matched] + field_surface = "population value" if words & {"population", "populations"} else "value" + normalized = f"Compare {field_surface} for {keys[0]} versus {keys[1]}; by how much do they differ?" + self._last_tool_normalization = { + "schema_version": "oracle-stage5k20-tool-row-grounding-v1", + "operation": "compare", + "expert_id": expert.schema.expert_id, + "row_keys": keys, + "original_question_sha256": hashlib.sha256(clean(text).encode("utf-8")).hexdigest(), + "normalized_question_sha256": hashlib.sha256(normalized.encode("utf-8")).hexdigest(), + "schema_values_only": True, + "answer_values_used": False, + } + return normalized + return text + + def _handle_tool(self, session_id: str, text: str) -> HandlerResult | None: + result = super()._handle_tool(session_id, self._normalize_tool_question(text)) + grounding = self._last_tool_normalization + if not ( + result + and result.status == "answered" + and grounding + and grounding.get("operation") == "compare" + and result.tool_audit + and self.tool_backend is not None + and hasattr(self.tool_backend, "experts") + ): + return result + expert = getattr(self.tool_backend, "experts", {}).get(grounding["expert_id"]) + field = result.tool_audit.get("field") + if expert is None or not field: + return result + ordered_values: list[str] = [] + for expected_key in grounding["row_keys"]: + row = next(( + value for value in expert.rows + if "|".join(clean(value[key]) for key in expert.schema.primary_key) == expected_key + ), None) + if row is None: + return result + ordered_values.append(display_number(row[str(field)])) + if len(ordered_values) != 2: + return result + difference = abs(Decimal(ordered_values[0]) - Decimal(ordered_values[1])) + ordered_values.append(display_number(str(difference))) + tool_audit = dict(result.tool_audit) + tool_audit["values"] = ordered_values + tool_audit["compare_order_source"] = "surface_mention_order" + revised_result = tuple( + (key, ordered_values if key == "values" else value) + for key, value in result.result + ) + return HandlerResult( + result.operation, + result.status, + result.arguments, + revised_result, + result.clauses, + result.evidence, + (*result.computation_trace, "restore_surface_mention_order"), + tool_audit=tool_audit, + memory_audit=result.memory_audit, + ) + + @classmethod + def _construction_frame(cls, text: str, context: Mapping[str, Any]) -> ConstructionFrame: + transfer = cls._parse_transfer(text) + if transfer: + return transfer[0] + state = cls._parse_state_contrast(text) + if state: + return state[0] + causal = cls._parse_causal_chain(text) + if causal: + return causal[0] + arithmetic = cls._bound_arithmetic(text) + if arithmetic: + _value, _display, expression = arithmetic + return ConstructionFrame( + "numeric_construction", "CALCULATE", "bounded_arithmetic", + (("expression", expression),), f"CALCULATE({expression})", + ) + if context.get("training_corpus"): + return ConstructionFrame("expert_construction", "COMPILE_CORPUS_EXPERT", "typed_corpus_context") + words = set(tokens(text)) + if words & {"population", "populations", "earthquake", "earthquakes", "catalog", "database"}: + operation = "COMPARE" if words & {"compare", "differ"} else "COUNT" if {"how", "many"} <= words else "BOUNDED_TOOL" + return ConstructionFrame("tool_construction", operation, "schema_grounded_tool_request") + if re.search(r"\bwhat\s+does\s+(?:it|this|that)\b", clean(text), re.IGNORECASE): + return ConstructionFrame("discourse_construction", "RESOLVE_AND_QUERY_ROLE", "anaphoric_role_query") + return ConstructionFrame("open_construction", "UNRESOLVED", "no_finite_operator_match") + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ) -> dict[str, Any]: + local_context = dict(context or {}) + frame = self._construction_frame(text, local_context) + self._last_tool_normalization = None + output = super().answer( + session_id, + text, + local_context, + ablate_specialists=ablate_specialists, + ) + output["schema_version"] = SCHEMA + output["construction_audit"] = frame.as_dict() + output["construction_audit"]["executed_operation"] = output["operation"] + output["construction_audit"]["executed_operation_family"] = output["operation_family"] + output["construction_audit"]["tool_row_grounding"] = self._last_tool_normalization + output["construction_audit"]["pre_malkuth_payload_unbinds"] = 0 + output["runtime"]["construction_language_model_calls"] = 0 + output["runtime"]["construction_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["CompositionalConstructionNativeAgent", "ConstructionFrame"] diff --git a/root/wander-train/stage5k221a_realizing_conversational_kernel.py b/root/wander-train/stage5k221a_realizing_conversational_kernel.py new file mode 100644 index 0000000000000000000000000000000000000000..857131dd1e49e3cf9edc57c5270d85f762765e96 --- /dev/null +++ b/root/wander-train/stage5k221a_realizing_conversational_kernel.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""K221 attach K220 realization strictly after K218/Malkuth release.""" +from __future__ import annotations + +from pathlib import Path +from typing import Sequence + +from stage5k205a_geometric_causal_kernel import _sha_json +from stage5k218a_compiler_authoritative_geometric_kernel import load_compiler_authoritative_kernel +from stage5k220a_evidence_preserving_realizer import EvidencePreservingRealizer + + +SCHEMA = "oracle-stage5k221a-realizing-conversational-kernel-v1" + + +class RealizingConversationalKernel: + def __init__(self, bundle_path: Path, *, ablations: Sequence[str] = ()): + self.authority = load_compiler_authoritative_kernel(bundle_path, ablations=ablations) + self.realizer = EvidencePreservingRealizer() + + def answer(self, text: str): + result = self.authority.answer(text) + original_response_sha256 = _sha_json(result.get("response", "")) + realized = self.realizer.realize(text, result).public() + for clause, sentence in zip( + (value for value in result["clauses"] if value.get("released")), + realized["clause_sentences"], + ): + clause["response"] = sentence + result["schema_version"] = SCHEMA + result["operation"] = "typed_proposition_geometric_authority_post_malkuth_realization" + result["response"] = realized["response"] + result["realization"] = realized + result["audit"]["authority_response_sha256_before_realization"] = original_response_sha256 + result["audit"]["realizer_input_boundary"] = "released_post_malkuth_propositions_only" + result.pop("turn_manifest_sha256", None) + result["turn_manifest_sha256"] = _sha_json(result) + return result + + +def load_realizing_kernel(bundle_path: Path, *, ablations: Sequence[str] = ()) -> RealizingConversationalKernel: + return RealizingConversationalKernel(bundle_path, ablations=ablations) + + +__all__ = ["RealizingConversationalKernel", "SCHEMA", "load_realizing_kernel"] diff --git a/root/wander-train/stage5k223a_identity_anchored_session.py b/root/wander-train/stage5k223a_identity_anchored_session.py new file mode 100644 index 0000000000000000000000000000000000000000..c1c019b68720c2ff412899dfe35b2d11884dcc85 --- /dev/null +++ b/root/wander-train/stage5k223a_identity_anchored_session.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""K223 canonical Oracle identity as the protected root of every turn state.""" +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from typing import Any, Mapping, Sequence + +import numpy as np + +from train_stage2a_role_ladder import HRR_DIM, bind, cosine, normalize, unitary + + +SCHEMA = "oracle-stage5k223a-identity-anchored-session-v1" + + +def canonical_sha(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class OracleIdentity: + name: str = "Oracle" + kind: str = "CPU-native relational language and cognitive runtime" + purpose: str = "compile natural-language requests into authenticated relational states, reason through bounded specialist knowledge, and realize evidence-preserving answers" + architecture: str = "HRR binding, four-head geometric authority, ten-node and twenty-two-path routing, a four-to-one stabilizing spine, and Malkuth-only semantic release" + epistemic_rule: str = "state what authenticated evidence supports, preserve provenance, and expose uncertainty or missing authority rather than fabricate" + current_scope: str = "typed catalog-grounded propositions and persistent bounded conversation; unrestricted language, broad world knowledge, and autonomous agency remain under construction" + + def contract(self) -> dict[str, str]: + return asdict(self) + + @property + def sha256(self) -> str: + return canonical_sha(self.contract()) + + @property + def root(self) -> np.ndarray: + return unitary("k223:oracle-identity:" + self.sha256) + + def introduction(self) -> str: + return ( + f"I am {self.name}, a {self.kind}. My purpose is to {self.purpose}. " + f"I follow one epistemic rule: {self.epistemic_rule}." + ) + + def capability_statement(self) -> str: + return ( + "I can compile active, passive, wh-variable, modal, negative, and chained propositions; " + "route authenticated specialist facts through the geometric authority; preserve source provenance; " + "and maintain bounded conversational context. " + f"My present boundary is {self.current_scope}." + ) + + +class IdentityStateProjector: + def __init__(self, identity: OracleIdentity | None = None): + self.identity = identity or OracleIdentity() + self._roles = { + name: unitary("k223:turn-role:" + name) + for name in ("session", "turn", "input", "output", "specialist", "salience") + } + + def project( + self, + *, + session_id: str, + turn_index: int, + input_text: str, + output_text: str, + active_specialist: str | None, + salience: Sequence[str], + ) -> dict[str, Any]: + values: Mapping[str, str] = { + "session": session_id, + "turn": str(turn_index), + "input": hashlib.sha256(input_text.encode("utf-8")).hexdigest(), + "output": hashlib.sha256(output_text.encode("utf-8")).hexdigest(), + "specialist": active_specialist or "none", + "salience": "|".join(salience) or "none", + } + payload = np.zeros(HRR_DIM, dtype=np.complex64) + for role, value in values.items(): + payload += bind(self._roles[role], unitary("k223:turn-value:" + value)) + payload = normalize(payload) + root = self.identity.root + residual = payload - np.complex64(np.vdot(root, payload)) * root + protected = normalize(np.float32(4.0) * root + residual) + return { + "schema_version": SCHEMA, + "identity_contract": self.identity.contract(), + "identity_sha256": self.identity.sha256, + "identity_is_turn_root": True, + "protected_identity_root_cosine": cosine(root, protected), + "turn_state_sha256": hashlib.sha256(np.ascontiguousarray(protected).tobytes()).hexdigest(), + "payload_roles": list(values), + "pre_malkuth_identity_unbinds": 0, + "traditional_system_prompt_overlay": False, + "traditional_language_model": False, + } + + +__all__ = ["IdentityStateProjector", "OracleIdentity", "SCHEMA", "canonical_sha"] diff --git a/root/wander-train/stage5k227a_implication_aware_discourse_runtime.py b/root/wander-train/stage5k227a_implication_aware_discourse_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..7ad32b27669548f0af05f8a7910230810acf113c --- /dev/null +++ b/root/wander-train/stage5k227a_implication_aware_discourse_runtime.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""K227 expose only verified K226 paths through the K224 discourse runtime.""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Mapping + +from stage5k205a_geometric_causal_kernel import _sha_json +from stage5k224a_persistent_discourse_runtime import JsonSessionStore, PersistentDiscourseRuntime, human +from stage5k226a_verified_hrr_implication_substrate import VerifiedHRRImplicationSubstrate + + +SCHEMA = "oracle-stage5k227a-implication-aware-discourse-runtime-v1" +REASONING_CUES = re.compile(r"\b(how|path|chain|connect|connected|relate|related|ultimately|indirectly)\b", re.I) + + +class ImplicationAwareDiscourseRuntime: + def __init__( + self, + bundles: Mapping[str, Path], + *, + session_store: JsonSessionStore | None = None, + substrate_checkpoint: Path | None = None, + saturate: bool = True, + ): + self.discourse = PersistentDiscourseRuntime(bundles, session_store=session_store) + self.substrate = VerifiedHRRImplicationSubstrate(bundles, checkpoint_path=substrate_checkpoint) + if saturate: + self.substrate.run_until_stable(maximum_hops=4) + entities = {edge.source for edge in self.substrate.edges.values()} | {edge.target for edge in self.substrate.edges.values()} + self.entities = tuple(sorted(entities, key=lambda value: (-len(human(value)), value))) + + def _entity_occurrences(self, text: str): + lowered = text.casefold() + values = [] + for entity in self.entities: + surface = human(entity) + for match in re.finditer(r"(? tuple[str, list[dict]]: + sentences = [] + plan = [] + for ordinal, edge in enumerate(implication.evidence_ledger): + transition = "First" if ordinal == 0 else ("Finally" if ordinal == len(implication.evidence_ledger) - 1 else "Then") + sentence = f"{transition}, {human(edge['source'])} connects to {human(edge['target'])} through “{human(edge['relation'])}”." + sentences.append(sentence) + plan.append({ + "ordinal": ordinal, + "fact_id": edge["fact_id"], + "source": edge["source"], + "relation": edge["relation"], + "target": edge["target"], + "discourse_function": "verified_implication_step", + "sentence": sentence, + }) + response = f"The verified implication is a {len(sentences)}-step authenticated path. " + " ".join(sentences) + return response, plan + + def answer(self, session_id: str, text: str): + state = self.discourse._state(session_id) + user_text = " ".join(str(text).strip().split()) + if not REASONING_CUES.search(user_text): + return self.discourse.answer(session_id, user_text) + resolved, antecedent = self.discourse._resolve_pronoun(user_text, state) + occurrences = self._entity_occurrences(resolved) + if len(occurrences) < 2: + return self.discourse.answer(session_id, user_text) + source = occurrences[0][2] + target = occurrences[-1][2] + paths = self.substrate.find(source, target) + if not paths: + bundle = self.discourse._route(resolved, state.active_bundle) + return self.discourse._finalize( + state, + user_text=user_text, + resolved_text=resolved, + status="clarification", + operation="verified_implication_lookup", + response=f"I found no verified authenticated path from {human(source)} to {human(target)} in the current substrate.", + bundle=bundle, + plan=[], + evidence=[], + coreference=antecedent, + ) + implication = paths[0] + response, plan = self._response(implication) + evidence = list(implication.evidence_ledger) + bundle = evidence[0]["bundle_name"] if evidence else state.active_bundle + state.active_bundle = bundle + state.last_plan = list(plan) + state.last_evidence = list(evidence) + self.discourse._update_salience(state, plan) + result = self.discourse._finalize( + state, + user_text=user_text, + resolved_text=resolved, + status="answered", + operation="verified_hrr_implication_path", + response=response, + bundle=bundle, + plan=plan, + evidence=evidence, + coreference=antecedent, + ) + result["schema_version"] = SCHEMA + result["implication"] = implication.public() + result["audit"]["known_fact_promoted"] = False + result["audit"]["implication_verifier_status"] = implication.verifier_status + result.pop("turn_manifest_sha256", None) + result["turn_manifest_sha256"] = _sha_json(result) + return result + + +__all__ = ["ImplicationAwareDiscourseRuntime", "SCHEMA"] diff --git a/root/wander-train/stage5k22_typed_intent_precedence_agent.py b/root/wander-train/stage5k22_typed_intent_precedence_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..3f7213f396c038132364c5da573dd91975a65d33 --- /dev/null +++ b/root/wander-train/stage5k22_typed_intent_precedence_agent.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""K22: typed intent precedence over K20's construction algebra. + +K21 showed one remaining dispatch collision: a schema-grounded question whose +surface began with "what is" was consumed as a lexical definition query before +the bounded dataset handler could test the named row. K22 makes handler +exclusion explicit. Definition constructions are eligible only when the input +is not already typed as arithmetic, dataset, provenance, corpus, evidence, or +sensor work. No answer labels, model calls, or learned carrier are added. +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, digest, tokens +from stage5k20_compositional_construction_agent import ( + CompositionalConstructionNativeAgent, +) + + +SCHEMA = "oracle-stage5k22-typed-intent-precedence-agent-v1" +DATASET_WORDS = frozenset({ + "population", "populations", "earthquake", "earthquakes", "catalog", + "database", "observation", "observations", "record", "records", + "lookup", "gdp", "magnitude", "depth", "deepest", "source", "license", + "provenance", "publisher", "published", +}) + + +class TypedIntentPrecedenceNativeAgent(CompositionalConstructionNativeAgent): + def __init__(self, *args: Any, **kwargs: Any) -> None: + kwargs.setdefault("agent_id", "oracle-stage5k22-typed-intent-precedence-agent") + super().__init__(*args, **kwargs) + self._last_intent_precedence: dict[str, Any] = {} + + @staticmethod + def _typed_intent_family(text: str, context: Mapping[str, Any] | None = None) -> str: + local = dict(context or {}) + words = set(tokens(text)) + if local.get("sensor_stream"): + return "sensor" + if local.get("training_corpus"): + return "corpus_expert" + if local.get("evidence"): + return "evidence_synthesis" + if words & DATASET_WORDS: + return "bounded_dataset" + if words & { + "calculate", "calculator", "compute", "multiply", "multiplied", + "times", "twice", "divide", "divided", "plus", "minus", + }: + return "bounded_arithmetic" + return "definition_or_event_or_open" + + def _handle_definition(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + family = self._typed_intent_family(text) + excluded = family in { + "bounded_dataset", "bounded_arithmetic", "corpus_expert", + "evidence_synthesis", "sensor", + } + self._last_intent_precedence = { + "schema_version": "oracle-stage5k22-intent-precedence-audit-v1", + "selected_intent_family": family, + "definition_handler_excluded": excluded, + "exclusion_is_answer_independent": True, + "answer_or_gold_labels_used": False, + } + if excluded: + return None + return super()._handle_definition(session_id, text, turn_id) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ) -> dict[str, Any]: + local_context = dict(context or {}) + family = self._typed_intent_family(text, local_context) + self._last_intent_precedence = { + "schema_version": "oracle-stage5k22-intent-precedence-audit-v1", + "selected_intent_family": family, + "definition_handler_excluded": family != "definition_or_event_or_open", + "exclusion_is_answer_independent": True, + "answer_or_gold_labels_used": False, + } + output = super().answer( + session_id, + text, + local_context, + ablate_specialists=ablate_specialists, + ) + output["schema_version"] = SCHEMA + output["intent_precedence_audit"] = dict(self._last_intent_precedence) + output["intent_precedence_audit"]["executed_operation"] = output["operation"] + output["intent_precedence_audit"]["pre_malkuth_payload_unbinds"] = 0 + output["runtime"]["intent_classifier_model_calls"] = 0 + output["runtime"]["intent_classifier_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["TypedIntentPrecedenceNativeAgent"] diff --git a/root/wander-train/stage5k230a_typed_agentic_runtime.py b/root/wander-train/stage5k230a_typed_agentic_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..6b1fef811bdcac6f5c8ef3a1f6d45d0fbe81eeeb --- /dev/null +++ b/root/wander-train/stage5k230a_typed_agentic_runtime.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""K230 bounded natural-language planning over the K229 typed tool ABI.""" +from __future__ import annotations + +import hashlib +import re +from pathlib import Path +from typing import Mapping + +from stage5k224a_persistent_discourse_runtime import JsonSessionStore +from stage5k227a_implication_aware_discourse_runtime import ImplicationAwareDiscourseRuntime +from stage5k229a_typed_tool_abi import ToolCall, ToolSpec, TypedToolRegistry, calculator_spec, canonical_sha + + +SCHEMA = "oracle-stage5k230a-typed-agentic-runtime-v1" +SPLIT = re.compile(r"\s*(?:,\s*)?(?:and\s+then|then)\s+|\s*;\s*", re.I) +CALCULATE = re.compile(r"\b(?:calculate|compute|evaluate)\s+(.+)$", re.I) + + +class TypedAgenticRuntime: + def __init__( + self, + bundles: Mapping[str, Path], + *, + session_store: JsonSessionStore | None = None, + substrate_checkpoint: Path | None = None, + maximum_tool_calls: int = 4, + ): + self.reasoning = ImplicationAwareDiscourseRuntime(bundles, session_store=session_store, substrate_checkpoint=substrate_checkpoint) + self.maximum_tool_calls = maximum_tool_calls + self.registry = TypedToolRegistry() + self.registry.register(calculator_spec()) + self.registry.register(ToolSpec( + "list_specialists", "List authenticated specialists loaded in this runtime.", {}, {"specialists": "array"}, "read_only", False, + lambda _arguments: {"specialists": sorted(self.reasoning.discourse.kernels)}, + )) + self.registry.register(ToolSpec( + "substrate_stats", "Read counts from the verified implication substrate.", {}, {"known_facts": "integer", "verified_implications": "integer", "cycles": "integer"}, "read_only", False, + lambda _arguments: { + "known_facts": len(self.reasoning.substrate.edges), + "verified_implications": len(self.reasoning.substrate.implications), + "cycles": self.reasoning.substrate.cycle_count, + }, + )) + + @staticmethod + def _clean_expression(value: str) -> str: + return value.strip().rstrip(".?!") + + def plan(self, text: str) -> dict: + source = " ".join(str(text).strip().split()) + segments = [value.strip(" ,") for value in SPLIT.split(source) if value.strip(" ,")] + calls = [] + unresolved = [] + for ordinal, segment in enumerate(segments): + calculate = CALCULATE.search(segment) + if calculate: + arguments = {"expression": self._clean_expression(calculate.group(1))} + tool_name = "calculator" + elif re.search(r"\b(list|show|name)\b.*\b(specialists|experts)\b", segment, re.I): + arguments = {} + tool_name = "list_specialists" + elif re.search(r"\b(how many|count|statistics|stats)\b.*\b(implications|facts|substrate)\b", segment, re.I): + arguments = {} + tool_name = "substrate_stats" + else: + unresolved.append({"ordinal": ordinal, "segment": segment, "reason": "no_typed_tool_intent"}) + continue + body = {"ordinal": ordinal, "tool_name": tool_name, "arguments": arguments} + calls.append(ToolCall( + call_id="call_" + canonical_sha(body)[:20], + tool_name=tool_name, + arguments=arguments, + side_effect_class=next(value["side_effect_class"] for value in self.registry.specs() if value["name"] == tool_name), + authorization="preauthorized_pure_or_read_only", + )) + return { + "schema_version": SCHEMA, + "source_text_sha256": hashlib.sha256(source.encode("utf-8")).hexdigest(), + "calls": [value.public() for value in calls], + "unresolved_segments": unresolved, + "maximum_tool_calls": self.maximum_tool_calls, + "plan_authorized": bool(calls) and len(calls) <= self.maximum_tool_calls and not unresolved, + "traditional_language_model": False, + "next_token_predictions": 0, + } + + @staticmethod + def _realize(results: list[dict]) -> str: + sentences = [] + for value in results: + if value["status"] != "succeeded": + sentences.append(f"The {value.get('tool_name', 'requested')} tool was not executed: {value.get('reason', 'validation failed')}.") + continue + if value["tool_name"] == "calculator": + output = value["output"] + sentences.append(f"The calculator returns {output['value']} for {output['expression']}.") + elif value["tool_name"] == "list_specialists": + sentences.append("The loaded specialists are " + ", ".join(value["output"]["specialists"]) + ".") + elif value["tool_name"] == "substrate_stats": + output = value["output"] + sentences.append(f"The verified substrate contains {output['known_facts']} known facts and {output['verified_implications']} verified implications after {output['cycles']} cycles.") + return " ".join(sentences) + + def answer(self, session_id: str, text: str): + plan = self.plan(text) + if not plan["calls"]: + return self.reasoning.answer(session_id, text) + state = self.reasoning.discourse._state(session_id) + if not plan["plan_authorized"]: + response = "I recognized a tool request, but its complete typed plan could not be authorized within the bounded tool grammar." + return self.reasoning.discourse._finalize(state, user_text=text, resolved_text=text, status="clarification", operation="typed_tool_planning", response=response, bundle=state.active_bundle, plan=[], evidence=[], coreference=None) + results = [self.registry.call(value["tool_name"], value["arguments"]) for value in plan["calls"]] + response = self._realize(results) + evidence = [{"tool_name": value.get("tool_name"), "result_sha256": value.get("result_sha256"), "side_effect_class": value.get("side_effect_class"), "output": value.get("output")} for value in results if value["status"] == "succeeded"] + status = "answered" if all(value["status"] == "succeeded" for value in results) else "clarification" + result = self.reasoning.discourse._finalize(state, user_text=text, resolved_text=text, status=status, operation="typed_tool_execution", response=response, bundle=state.active_bundle, plan=[], evidence=evidence, coreference=None) + result["schema_version"] = SCHEMA + result["tool_plan"] = plan + result["tool_results"] = results + result["audit"]["tool_calls"] = len(results) + result["audit"]["external_writes"] = sum(value.get("side_effect_class") == "external_write" and value["status"] == "succeeded" for value in results) + result["audit"]["bounded_agent_loop"] = True + result.pop("turn_manifest_sha256", None) + result["turn_manifest_sha256"] = canonical_sha(result) + return result + + +__all__ = ["SCHEMA", "TypedAgenticRuntime"] diff --git a/root/wander-train/stage5k235a_oracle_relational_v3.py b/root/wander-train/stage5k235a_oracle_relational_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..d767bf62aaa9e8d15a6fc248c154596cf0af8cb7 --- /dev/null +++ b/root/wander-train/stage5k235a_oracle_relational_v3.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""K235 consolidated public API for the K214-K234 relational-v3 runtime.""" +from __future__ import annotations + +import tempfile +import time +from pathlib import Path +from typing import Any, Mapping + +from stage5k214a_authenticated_specialist_factory import compile_specialist_bundle +from stage5k224a_persistent_discourse_runtime import JsonSessionStore +from stage5k233a_hybrid_specialist_agentic_runtime import HybridSpecialistAgenticRuntime + + +VERSION = "0.7.0rc1" +SCHEMA = "oracle-stage5k235a-relational-v3-public-api-v1" + + +class OracleRelationalV3: + def __init__( + self, + bundles: Mapping[str, Path] | None = None, + *, + state_root: Path | None = None, + workspace_root: Path | None = None, + ): + root = Path(workspace_root).resolve() if workspace_root else Path(__file__).resolve().parents[1] + phase = root / "fable-content" / "oracle-phase3-2026-07-21" + self.workspace_root = root + self.phase_root = phase + self.bundle_paths = dict(bundles or { + "thermal": phase / "stage5k214_thermal_control_development_bundle_v1.json", + "grid": phase / "stage5k216b_grid_operations_transfer_bundle_v1.json", + "materials": phase / "stage5k232_materials_reliability_bundle_v1.json", + }) + if state_root is None: + state_root = Path(tempfile.mkdtemp(prefix="oracle-relational-v3-")) + self.state_root = Path(state_root).resolve() + self.state_root.mkdir(parents=True, exist_ok=True) + self.session_store = JsonSessionStore(self.state_root / "sessions") + started = time.perf_counter() + self.runtime = HybridSpecialistAgenticRuntime( + self.bundle_paths, + session_store=self.session_store, + substrate_checkpoint=self.state_root / "implication_substrate.json", + ) + self.startup_milliseconds = (time.perf_counter() - started) * 1000.0 + + def chat(self, text: str, *, session_id: str = "default") -> dict[str, Any]: + return self.runtime.answer(session_id, text) + + def capabilities(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA, + "version": VERSION, + "identity_rooted_persistent_dialogue": True, + "active_passive_wh_and_coordinated_propositions": True, + "post_malkuth_evidence_preserving_realization": True, + "verified_implication_hops": 4, + "typed_tools": ["calculator", "list_specialists", "substrate_stats"], + "tool_side_effect_scope": ["pure", "read_only"], + "maximum_ephemeral_specialists": 3, + "rapid_authenticated_specialist_compilation": True, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "unrestricted_conversation": False, + "unrestricted_reasoning": False, + "unrestricted_agency": False, + "boundary": "Catalog-grounded relational language, verified graph paths, bounded discourse, pure/read-only tools, and temporary specialist coordination.", + } + + def health(self) -> dict[str, Any]: + substrate = self.runtime.agent.reasoning.substrate + return { + "schema_version": SCHEMA, + "status": "ready", + "version": VERSION, + "startup_milliseconds": self.startup_milliseconds, + "bundles": sorted(self.bundle_paths), + "known_facts": len(substrate.edges), + "verified_implications": len(substrate.implications), + "substrate_cycles": substrate.cycle_count, + "state_root": str(self.state_root), + } + + def compile_and_load_specialist( + self, + name: str, + spec_path: Path, + *, + output_root: Path, + frozen_carrier_manifest: Path | None = None, + ) -> dict[str, Any]: + safe = str(name).strip() + if not safe or safe in self.bundle_paths: + raise ValueError("specialist name must be new and non-empty") + output_root = Path(output_root).resolve() + carrier = Path(frozen_carrier_manifest).resolve() if frozen_carrier_manifest else self.phase_root / "stage5k213_frozen_k212_construction_kernel_manifest_v1.json" + bundle_path = output_root / f"{safe}_bundle.json" + bundle = compile_specialist_bundle( + Path(spec_path), + frozen_carrier_manifest=carrier, + output_dir=output_root / "checkpoints", + bundle_path=bundle_path, + ) + self.bundle_paths[safe] = bundle_path + self.runtime = HybridSpecialistAgenticRuntime( + self.bundle_paths, + session_store=self.session_store, + substrate_checkpoint=self.state_root / f"implication_substrate_{len(self.bundle_paths)}.json", + ) + return { + "status": "loaded", + "name": safe, + "bundle_path": str(bundle_path), + "bundle_sha256": bundle.file_sha256, + "facts": len(bundle.facts), + "carrier_parameters_trained": 0, + "compile_seconds": bundle.payload["performance"]["total_compile_seconds"], + } + + +__all__ = ["OracleRelationalV3", "SCHEMA", "VERSION"] diff --git a/root/wander-train/stage5k238a_epistemically_guarded_release_candidate.py b/root/wander-train/stage5k238a_epistemically_guarded_release_candidate.py new file mode 100644 index 0000000000000000000000000000000000000000..4ad1b7dea85f68e05e34dcb89c3877d225267fec --- /dev/null +++ b/root/wander-train/stage5k238a_epistemically_guarded_release_candidate.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""K238 repair of intent precedence and cross-specialist argument erasure.""" +from __future__ import annotations + +import re +import types +from pathlib import Path + +from stage5k233a_hybrid_specialist_agentic_runtime import HybridSpecialistAgenticRuntime +from stage5k235a_oracle_relational_v3 import OracleRelationalV3 +from stage5k236b_punctuation_invariant_release_candidate import _punctuation_invariant_route + + +SCHEMA = "oracle-stage5k238a-epistemically-guarded-release-candidate-v1" + + +def _normalized(value: str) -> str: + return " ".join(re.findall(r"[a-z0-9]+(?:'[a-z0-9]+)?", str(value).casefold())) + + +def _specific_identity_act(_discourse, text: str) -> str | None: + """Resolve the more specific capability act before generic identity forms.""" + normalized = _normalized(text) + if any( + value in normalized + for value in ("what can you do", "your capabilities", "your purpose") + ): + return "capability" + if any( + value in normalized + for value in ("who are you", "what are you", "your identity", "your name") + ): + return "identity" + return None + + +class EpistemicallyGuardedHybridRuntime(HybridSpecialistAgenticRuntime): + """Withhold a single-clause answer when its explicit entities cross catalogs.""" + + def _mentioned_bundles(self, text: str) -> tuple[str, ...]: + normalized = " " + _normalized(text) + " " + discourse = self.agent.reasoning.discourse + return tuple( + sorted( + name + for name, kernel in discourse.kernels.items() + if any( + " " + _normalized(label.replace("_", " ")) + " " in normalized + for label in ( + *kernel.authority.pool._source_labels, + *kernel.authority.pool._target_labels, + ) + ) + ) + ) + + def answer(self, session_id: str, text: str): + components = self._components(text) + mentioned = self._mentioned_bundles(text) + if ( + len(components) == 1 + and components[0]["kind"] == "specialist" + and len(mentioned) > 1 + ): + discourse = self.agent.reasoning.discourse + state = discourse._state(session_id) + response = ( + "This clause combines explicit entities from multiple authenticated " + "specialists, but no authenticated cross-specialist proposition supports " + "that relation. Please separate the requests or provide cross-domain evidence." + ) + result = discourse._finalize( + state, + user_text=text, + resolved_text=text, + status="clarification", + operation="cross_specialist_argument_guard", + response=response, + bundle=None, + plan=[], + evidence=[], + coreference=None, + ) + result["schema_version"] = SCHEMA + result["audit"]["mentioned_specialists"] = list(mentioned) + result["audit"]["explicit_argument_erasure_prevented"] = True + result["audit"]["external_writes"] = 0 + return result + return super().answer(session_id, text) + + +class EpistemicallyGuardedOracleRelationalV3(OracleRelationalV3): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.runtime = EpistemicallyGuardedHybridRuntime( + self.bundle_paths, + session_store=self.session_store, + substrate_checkpoint=self.state_root / "implication_substrate.json", + ) + discourse = self.runtime.agent.reasoning.discourse + discourse._route = types.MethodType(_punctuation_invariant_route, discourse) + discourse._identity_act = types.MethodType(_specific_identity_act, discourse) + + +__all__ = [ + "EpistemicallyGuardedHybridRuntime", + "EpistemicallyGuardedOracleRelationalV3", + "SCHEMA", +] diff --git a/root/wander-train/stage5k23_contextual_intent_lattice_agent.py b/root/wander-train/stage5k23_contextual_intent_lattice_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..93b7e32cb0e0cd95e515c72d13993dabe26491a9 --- /dev/null +++ b/root/wander-train/stage5k23_contextual_intent_lattice_agent.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""K23: whole-construction intent lattice over K22. + +An isolated content word does not determine an operation. K23 ranks mutually +exclusive construction hypotheses before specialist routing: explicit local +definition, arithmetic program, event/state/causal frame, discourse query, +schema-grounded tool request, or open request. Thus "catalog" can be a locally +defined term, a theme in a transfer, or a dataset anchor depending on the whole +shape. The lattice is finite, inspectable, answer-independent, and model-free. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import clean, digest, tokens +from stage5k22_typed_intent_precedence_agent import TypedIntentPrecedenceNativeAgent + + +SCHEMA = "oracle-stage5k23-contextual-intent-lattice-agent-v1" + + +class ContextualIntentLatticeNativeAgent(TypedIntentPrecedenceNativeAgent): + def __init__(self, *args: Any, **kwargs: Any) -> None: + kwargs.setdefault("agent_id", "oracle-stage5k23-contextual-intent-lattice-agent") + super().__init__(*args, **kwargs) + self._last_intent_lattice: dict[str, Any] = {} + + @classmethod + def _intent_lattice( + cls, + text: str, + context: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + local = dict(context or {}) + surface = clean(text) + lower = surface.casefold() + words = set(tokens(surface)) + scores = { + "sensor": 1.0 if local.get("sensor_stream") else 0.0, + "corpus_expert": 1.0 if local.get("training_corpus") else 0.0, + "evidence_synthesis": 1.0 if local.get("evidence") else 0.0, + "explicit_definition": 1.0 if cls._definition(surface) else 0.0, + "bounded_arithmetic": 1.0 if cls._bound_arithmetic(surface) else 0.0, + "transfer_event": 1.0 if cls._parse_transfer(surface) else 0.0, + "state_contrast": 1.0 if cls._parse_state_contrast(surface) else 0.0, + "causal_chain": 1.0 if cls._parse_causal_chain(surface) else 0.0, + "caused_motion": 1.0 if re.search( + r"\b(?:drove|pushed|pulled|carried)\b.+\b(?:across|through|into|to)\b", + lower, + ) else 0.0, + "definition_role_query": 1.0 if re.search( + r"\bwhat\s+does\s+(?:a|an|the)?\s*(?:it|this|that|[a-z][a-z0-9_-]*)\s+[a-z]", + lower, + ) else 0.0, + "event_consequence_query": 1.0 if ( + re.search(r"\bwho\s+(?:has|holds|possesses)\b", lower) + or re.search(r"\bwhere\s+(?:is|did)\b", lower) + ) else 0.0, + "causal_query": 1.0 if "why" in words else 0.0, + "state_query": 1.0 if ( + words & {"which", "what", "contrast"} + and words & {"changed", "increased", "decreased", "rose", "fell"} + and words & {"fixed", "stable", "constant", "unchanged", "stayed", "remained"} + ) else 0.0, + "bounded_dataset": 0.0, + "definition_query": 0.0, + } + schema_anchors = { + "loaded", "database", "dataset", "catalog", "observation", "observations", + "row", "rows", "record", "records", "event", "events", "lookup", + "source", "license", "provenance", "publisher", "published", + } + data_domains = {"population", "populations", "earthquake", "earthquakes", "magnitude", "depth", "gdp"} + tool_forms = ( + bool(words & schema_anchors and words & data_domains) + or bool(words & {"compare", "versus", "differ"} and words & data_domains) + or ("how many" in lower and bool(words & {"record", "records", "event", "events", "observation", "observations"})) + or bool(words & {"deepest", "strongest", "largest", "maximum", "minimum"} and words & data_domains) + or bool(words & {"license", "provenance", "publisher", "published"} and words & {"catalog", "database", "source", "earthquake", "population"}) + or bool(re.search(r"\bpopulation\s+for\s+[a-z]", lower)) + ) + scores["bounded_dataset"] = 1.0 if tool_forms else 0.0 + definition_query = cls._definition_query(surface) + scores["definition_query"] = 1.0 if definition_query and not tool_forms else 0.0 + + precedence = ( + "sensor", "corpus_expert", "evidence_synthesis", + "explicit_definition", "bounded_arithmetic", "transfer_event", + "state_contrast", "causal_chain", "caused_motion", + "definition_role_query", "event_consequence_query", "state_query", + "bounded_dataset", "definition_query", "causal_query", + ) + selected = next((name for name in precedence if scores[name] > 0.0), "open") + return { + "schema_version": "oracle-stage5k23-intent-lattice-audit-v1", + "candidate_scores": scores, + "precedence": list(precedence), + "selected_intent_family": selected, + "selection_policy": "most_specific_whole_construction_then_typed_query", + "isolated_keyword_is_sufficient": False, + "answer_or_gold_labels_used": False, + "traditional_language_model": False, + "pretrained_embedding_encoder": False, + } + + @classmethod + def _typed_intent_family( + cls, + text: str, + context: Mapping[str, Any] | None = None, + ) -> str: + selected = cls._intent_lattice(text, context)["selected_intent_family"] + return { + "sensor": "sensor", + "corpus_expert": "corpus_expert", + "evidence_synthesis": "evidence_synthesis", + "bounded_arithmetic": "bounded_arithmetic", + "bounded_dataset": "bounded_dataset", + }.get(selected, "definition_or_event_or_open") + + @classmethod + def _routing_goal( + cls, + text: str, + context: Mapping[str, Any], + ) -> tuple[tuple[str, ...], ...]: + selected = cls._intent_lattice(text, context)["selected_intent_family"] + if selected == "sensor": + return (("sensor", "state"),) + if selected == "corpus_expert": + return (("corpus_expert",), ("linguistic", "synthesis")) + if selected == "evidence_synthesis": + return (("synthesis", "epistemic"),) + if selected in {"explicit_definition", "definition_role_query", "definition_query"}: + return (("definition", "linguistic"),) + if selected == "bounded_arithmetic": + return (("numeric", "tool_router"),) + if selected == "transfer_event" or selected == "event_consequence_query": + return (("event",), ("linguistic", "causal")) + if selected == "caused_motion": + return (("spatial", "event"),) + if selected == "state_contrast" or selected == "state_query": + return (("state", "contrast"),) + if selected in {"causal_chain", "causal_query"}: + return (("causal", "mechanism"),) + if selected == "bounded_dataset": + words = set(tokens(text)) + if words & {"license", "provenance", "publisher", "published", "source"}: + return (("provenance",),) + if words & {"compare", "differ", "count", "many", "maximum", "minimum", "deepest", "strongest"}: + return (("dataset",), ("numeric", "tool_router")) + return (("dataset",),) + return (("linguistic", "epistemic"),) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ) -> dict[str, Any]: + local_context = dict(context or {}) + lattice = self._intent_lattice(text, local_context) + self._last_intent_lattice = lattice + output = super().answer( + session_id, + text, + local_context, + ablate_specialists=ablate_specialists, + ) + output["schema_version"] = SCHEMA + output["intent_lattice_audit"] = dict(lattice) + output["intent_lattice_audit"]["executed_operation"] = output["operation"] + output["intent_lattice_audit"]["pre_malkuth_payload_unbinds"] = 0 + output["runtime"]["intent_lattice_model_calls"] = 0 + output["runtime"]["intent_lattice_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["ContextualIntentLatticeNativeAgent"] diff --git a/root/wander-train/stage5k25_causally_gated_intent_agent.py b/root/wander-train/stage5k25_causally_gated_intent_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..ffe104cc8b63e6457b58d794464c4ed4a16ba91a --- /dev/null +++ b/root/wander-train/stage5k25_causally_gated_intent_agent.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""K25: make K23's selected intent causally gate handler eligibility.""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, clean, digest, tokens +from stage5k23_contextual_intent_lattice_agent import ContextualIntentLatticeNativeAgent + + +SCHEMA = "oracle-stage5k25-causally-gated-intent-agent-v1" + + +class CausallyGatedIntentNativeAgent(ContextualIntentLatticeNativeAgent): + def __init__(self, *args: Any, **kwargs: Any) -> None: + kwargs.setdefault("agent_id", "oracle-stage5k25-causally-gated-intent-agent") + super().__init__(*args, **kwargs) + self._active_intent = "open" + self._dispatch_decisions: list[dict[str, Any]] = [] + + def _eligible(self, handler_family: str, allowed: set[str]) -> bool: + accepted = self._active_intent in allowed + self._dispatch_decisions.append({ + "handler_family": handler_family, + "selected_intent": self._active_intent, + "eligible": accepted, + }) + return accepted + + def _handle_arithmetic(self, text: str) -> HandlerResult | None: + if not self._eligible("arithmetic", {"bounded_arithmetic"}): + return None + return super()._handle_arithmetic(text) + + def _handle_definition(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + if not self._eligible( + "definition", + {"explicit_definition", "definition_query", "definition_role_query"}, + ): + return None + return super()._handle_definition(session_id, text, turn_id) + + def _handle_event(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + if not self._eligible( + "event_state_causal", + { + "transfer_event", "event_consequence_query", "caused_motion", + "state_contrast", "state_query", "causal_chain", "causal_query", + }, + ): + return None + return super()._handle_event(session_id, text, turn_id) + + def _mentioned_expert_ids(self, text: str) -> list[str]: + if self.tool_backend is None or not hasattr(self.tool_backend, "experts"): + return [] + lower = clean(text).casefold() + query = set(tokens(text)) + mentioned: list[str] = [] + for expert_id, expert in getattr(self.tool_backend, "experts", {}).items(): + lexicon = set(getattr(self.tool_backend, "expert_lexicons", {}).get(expert_id, ())) + domain_hit = ( + ("population" in query and "population" in lexicon) + or (bool(query & {"earthquake", "earthquakes"}) and "earthquake" in lexicon) + ) + row_hit = False + for row in expert.rows: + for field, value in row.items(): + candidates = expert.candidate_values.get(field, set()) + surface = clean(value).casefold() + if len(candidates) > 1 and len(surface) >= 4 and surface in lower: + row_hit = True + break + if row_hit: + break + if domain_hit or row_hit: + mentioned.append(expert_id) + return sorted(set(mentioned)) + + def _handle_tool(self, session_id: str, text: str) -> HandlerResult | None: + if not self._eligible("bounded_tool", {"bounded_dataset"}): + return None + words = set(tokens(text)) + mentioned = self._mentioned_expert_ids(text) + if len(mentioned) >= 2 and words & {"cause", "caused", "causal", "explain", "explained", "why"}: + result = self._abstention( + "bounded_tool", + clean(text), + "cross_expert_inference", + ) + return HandlerResult( + result.operation, + result.status, + result.arguments, + result.result, + result.clauses, + (("mentioned_expert_ids", mentioned),), + ( + "resolve_registered_schema_mentions", + "detect_cross_expert_join", + "withhold_unsupported_causal_join", + ), + memory_audit=result.memory_audit, + ) + return super()._handle_tool(session_id, text) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ) -> dict[str, Any]: + local_context = dict(context or {}) + lattice = self._intent_lattice(text, local_context) + self._active_intent = str(lattice["selected_intent_family"]) + self._dispatch_decisions = [] + output = super().answer( + session_id, + text, + local_context, + ablate_specialists=ablate_specialists, + ) + output["schema_version"] = SCHEMA + output["intent_dispatch_audit"] = { + "schema_version": "oracle-stage5k25-intent-dispatch-audit-v1", + "selected_intent_family": self._active_intent, + "handler_decisions": list(self._dispatch_decisions), + "selected_intent_causally_gates_handlers": True, + "mentioned_expert_ids": self._mentioned_expert_ids(text), + "pre_malkuth_payload_unbinds": 0, + "answer_or_gold_labels_used": False, + } + output["runtime"]["dispatch_model_calls"] = 0 + output["runtime"]["dispatch_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["CausallyGatedIntentNativeAgent"] diff --git a/root/wander-train/stage5k27_speech_act_lattice_agent.py b/root/wander-train/stage5k27_speech_act_lattice_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..2930bc67b2137e958292f363e42fee2d0b998978 --- /dev/null +++ b/root/wander-train/stage5k27_speech_act_lattice_agent.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""K27: explicit speech-act precedence over domain-content cues.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import clean, digest +from stage5k25_causally_gated_intent_agent import CausallyGatedIntentNativeAgent + + +SCHEMA = "oracle-stage5k27-speech-act-lattice-agent-v1" + + +class SpeechActLatticeNativeAgent(CausallyGatedIntentNativeAgent): + @classmethod + def _intent_lattice(cls, text: str, context: Mapping[str, Any] | None = None) -> dict[str, Any]: + local = dict(context or {}) + audit = super()._intent_lattice(text, local) + surface = clean(text).casefold() + explicit_creative = bool(re.match( + r"^(?:please\s+)?(?:write|compose|draft)\s+(?:a|an|the)?\s*" + r"(?:story|fable|poem|sonnet|comic|dialogue|scene)\b|" + r"^(?:please\s+)?tell\s+me\s+(?:a|an|the)\s+(?:story|fable)\b", + surface, + )) + bounded_context = any(local.get(key) for key in ("training_corpus", "evidence", "sensor_stream")) + audit = dict(audit) + scores = dict(audit["candidate_scores"]) + scores["creative_generation_request"] = 1.0 if explicit_creative and not bounded_context else 0.0 + audit["candidate_scores"] = scores + audit["explicit_speech_act_detected"] = explicit_creative + audit["bounded_context_overrides_creative_surface"] = explicit_creative and bounded_context + if explicit_creative and not bounded_context: + audit["selected_intent_family"] = "creative_generation_request" + audit["selection_policy"] = "explicit_speech_act_then_whole_construction_then_typed_query" + audit["schema_version"] = "oracle-stage5k27-speech-act-lattice-audit-v1" + return audit + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ) -> dict[str, Any]: + output = super().answer(session_id, text, context, ablate_specialists=ablate_specialists) + output["schema_version"] = SCHEMA + output["runtime"]["speech_act_model_calls"] = 0 + output["runtime"]["speech_act_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["SpeechActLatticeNativeAgent"] diff --git a/root/wander-train/stage5k28_native_generative_construction_agent.py b/root/wander-train/stage5k28_native_generative_construction_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..2493c1dd7f1e1762d36ca777e1fa14e35186e790 --- /dev/null +++ b/root/wander-train/stage5k28_native_generative_construction_agent.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""K28: native generative discourse constructions over K27. + +Creative requests compile to typed semantic graphs (fable, story, poem, or +dialogue) and pass through the same proof/QUBO-inverse/Malkuth path as every +other answer. Generation is whole-construction selection, not next-token +prediction. This is a finite productive grammar, not yet unrestricted prose. +""" + +from __future__ import annotations + +import hashlib +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest, tokens +from stage5k27_speech_act_lattice_agent import SpeechActLatticeNativeAgent + + +SCHEMA = "oracle-stage5k28-native-generative-construction-agent-v1" + + +def _without_article(value: str) -> str: + return re.sub(r"^(?:a|an|the)\s+", "", clean(value), flags=re.IGNORECASE) + + +class NativeGenerativeConstructionAgent(SpeechActLatticeNativeAgent): + @classmethod + def _routing_goal(cls, text: str, context: Mapping[str, Any]) -> tuple[tuple[str, ...], ...]: + if cls._intent_lattice(text, context)["selected_intent_family"] == "creative_generation_request": + return (("linguistic",), ("synthesis", "contrast")) + return super()._routing_goal(text, context) + + @staticmethod + def _event_capability_family(text: str) -> tuple[str, ...]: + if set(tokens(text)) & {"story", "fable", "poem", "sonnet", "dialogue", "scene", "comic"}: + return ("linguistic", "synthesis", "contrast") + return SpeechActLatticeNativeAgent._event_capability_family(text) + + @staticmethod + def _creative_roles(text: str) -> tuple[str, str, str]: + surface = clean(text).rstrip(".?!") + lower = surface.casefold() + form_match = re.search(r"\b(story|fable|poem|sonnet|dialogue|scene|comic)\b", lower) + form = form_match.group(1) if form_match else "story" + between = re.search(r"\bbetween\s+(.+?)\s+and\s+(.+)$", surface, re.IGNORECASE) + about_pair = re.search(r"\babout\s+(.+?)\s+and\s+(.+)$", surface, re.IGNORECASE) + pair = between or about_pair + if pair: + left, right = (_without_article(pair.group(index)) for index in (1, 2)) + else: + about = re.search(r"\babout\s+(.+)$", surface, re.IGNORECASE) + left = _without_article(about.group(1)) if about else "a changing pattern" + right = "patient observer" if form in {"story", "fable", "dialogue", "scene", "comic"} else "quiet horizon" + return form, left, right + + @classmethod + def _creative_clauses(cls, form: str, left: str, right: str) -> tuple[SemanticClause, ...]: + left_subject = "The " + left + right_subject = "The " + right + if form in {"fable", "story"}: + return ( + SemanticClause(left_subject, "encounter", object_value=right_subject.casefold(), tense="past"), + SemanticClause(left_subject, "race", qualifiers=(("toward", "an answer"),), tense="past"), + SemanticClause(right_subject, "remain", complement="quiet enough to notice the path", tense="past"), + SemanticClause(left_subject, "learn", object_value="that direction matters more than haste", tense="past"), + SemanticClause("The lesson", "be", complement="that patience can turn motion into meaning"), + ) + if form in {"dialogue", "scene", "comic"}: + return ( + SemanticClause(left_subject, "ask", object_value="what motion means"), + SemanticClause(right_subject, "answer", object_value="that relation gives motion direction"), + SemanticClause(left_subject, "offer", object_value="a changing pattern"), + SemanticClause(right_subject, "offer", object_value="a stable frame"), + SemanticClause("Their exchange", "form", object_value="one shared map"), + ) + return ( + SemanticClause(left_subject, "move", qualifiers=(("through", "the dark"),)), + SemanticClause(right_subject, "hold", object_value="the measure of the motion"), + SemanticClause("Each returning pattern", "become", complement="a path"), + SemanticClause("Each quiet interval", "become", complement="a place to listen"), + SemanticClause("The relation", "remain", complement="after the separate shapes pass"), + ) + + def _handle_event(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + if self._active_intent != "creative_generation_request": + return super()._handle_event(session_id, text, turn_id) + form, left, right = self._creative_roles(text) + clauses = self._creative_clauses(form, left, right) + return HandlerResult( + "creative_discourse", + "answered", + (("form", form), ("left_topic", left), ("right_topic", right)), + (("semantic_clauses", len(clauses)), ("whole_construction", form)), + clauses, + (("prompt_sha256", hashlib.sha256(clean(text).encode()).hexdigest()),), + ( + "parse_explicit_creative_speech_act", + "bind_topic_roles", + "instantiate_native_semantic_schema", + "compose_multi_clause_proof", + ), + ) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ) -> dict[str, Any]: + output = super().answer(session_id, text, context, ablate_specialists=ablate_specialists) + output["schema_version"] = SCHEMA + output["generative_construction_audit"] = { + "schema_version": "oracle-stage5k28-generative-construction-audit-v1", + "productive_family": output["operation"] == "creative_discourse", + "operation": output["operation"], + "whole_construction_inverse": bool(output["quadratic_inverse"].get("whole_construction_selected_globally")), + "next_token_predictions": 0, + "traditional_language_model": False, + "pretrained_embedding_encoder": False, + } + output["runtime"]["generative_model_calls"] = 0 + output["runtime"]["generative_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["NativeGenerativeConstructionAgent"] diff --git a/root/wander-train/stage5k29_general_knowledge_domains.py b/root/wander-train/stage5k29_general_knowledge_domains.py new file mode 100644 index 0000000000000000000000000000000000000000..a21eefaf48ac19e98974ccb82a47bed8d33d6eb3 --- /dev/null +++ b/root/wander-train/stage5k29_general_knowledge_domains.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""K29 source-grounded domain ontology for a general-knowledge specialist basis. + +The public C94 corpus supplies natural sentences and immutable provenance. K29 +adds only a coarse, auditable source-document ontology. Labels are never +inferred from validation text, and designated fit documents are held out in +their entirety. The ontology is deliberately broad: the experiment asks how +few separately addressable domain modes can cover a useful general-knowledge +surface, not how many academic departments can be named. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + + +SCHEMA = "oracle-stage5k29-general-knowledge-domain-corpus-v1" +ROOT = Path(__file__).resolve().parent.parent +PUBLIC_CORPUS = ( + ROOT / "fable-content" / "oracle-phase3-2026-07-21" + / "stage5c94_public_contextual_corpus_v1.jsonl" +) + + +@dataclass(frozen=True) +class KnowledgeDomainSpec: + domain_id: str + description: str + cues: tuple[str, ...] + + +DOMAIN_SPECS = ( + KnowledgeDomainSpec( + "physical_engineering", + "matter energy forces chemistry mechanisms measurement and engineered systems", + ("physics", "physical", "chemistry", "chemical", "matter", "energy", "force", "motion", "light", "electric", "mechanism", "machine", "device"), + ), + KnowledgeDomainSpec( + "earth_space", + "astronomy planetary science geology climate oceans and Earth systems", + ("planet", "planetary", "star", "orbit", "orbital", "galaxy", "earth", "rock", "geology", "climate", "ocean", "atmosphere", "environment"), + ), + KnowledgeDomainSpec( + "life_health", + "biology anatomy physiology microbiology evolution ecology and health", + ("cell", "organism", "ecosystem", "biology", "biomedical", "gene", "species", "body", "blood", "health", "disease", "microbe", "physiology", "evolution"), + ), + KnowledgeDomainSpec( + "mathematics_computation", + "number quantity algebra geometry functions proof algorithms and computation", + ("number", "mathematics", "mathematical", "equation", "function", "geometry", "ratio", "theorem", "proof", "calculate", "statistic", "statistical", "quantitative", "algorithm", "variable", "model"), + ), + KnowledgeDomainSpec( + "social_behavioral", + "psychology sociology anthropology behavior groups institutions and society", + ("behavior", "behavioral", "mind", "cognition", "psychology", "society", "social", "group", "culture", "identity", "institution", "collective"), + ), + KnowledgeDomainSpec( + "history_culture", + "historical events peoples civilizations material culture and change through time", + ("history", "historical", "period", "empire", "war", "century", "ancient", "king", "people", "civilization", "revolution"), + ), + KnowledgeDomainSpec( + "economics_organization", + "economics markets trade firms management business and collective production", + ("market", "markets", "price", "trade", "economy", "economic", "economics", "capital", "labor", "resource", "incentive", "business", "management", "firm", "consumer"), + ), + KnowledgeDomainSpec( + "governance_law", + "government politics law rights public policy constitutions and civic institutions", + ("government", "governance", "law", "policy", "regulation", "state", "rights", "constitution", "constitutional", "political", "authority", "election", "court", "citizen"), + ), + KnowledgeDomainSpec( + "philosophy_religion", + "philosophy ethics logic metaphysics epistemology religion and systems of value", + ("truth", "knowledge", "ethics", "ethical", "reason", "philosophy", "moral", "virtue", "religion", "religious", "god", "belief", "metaphysics"), + ), + KnowledgeDomainSpec( + "language_arts", + "language writing rhetoric narrative drama poetry literature and artistic expression", + ("language", "word", "write", "writing", "reader", "readable", "sentence", "prose", "story", "poem", "novel", "character", "speech", "literature", "interpret"), + ), +) + + +PHYSICAL_ENGINEERING = frozenset({ + "einstein_relativity.txt", "faraday_chemical_history_candle.txt", + "newton_opticks.txt", "newton_principia_translated.txt", + "chemistry-2e.txt", "chemistry-atoms-first-2e.txt", +}) +EARTH_SPACE = frozenset({ + "astronomy.txt", "lyell_principles_geology.txt", "darwin_voyage_beagle.txt", +}) +LIFE_HEALTH = frozenset({ + "darwin_origin.txt", "huxley_man_place_in_nature.txt", + "anatomy-and-physiology.txt", "biology-2e.txt", "microbiology.txt", + "concepts-biology.txt", +}) +MATHEMATICS_COMPUTATION = frozenset({ + "euclid_elements_book7_to_13.txt", "precalculus.txt", +}) +SOCIAL_BEHAVIORAL = frozenset({ + "introduction-anthropology.txt", "psychology-2e.txt", + "introduction-sociology-3e.txt", +}) +HISTORY_CULTURE = frozenset({ + "herodotus_histories.txt", "tacitus_annals.txt", + "thucydides_peloponnesian_war.txt", "world-history-volume-1.txt", + "world-history-volume-2.txt", "plutarch_lives_v1.txt", "us-history.txt", +}) +ECONOMICS_ORGANIZATION = frozenset({ + "smith_wealth_of_nations.txt", "introduction-business.txt", + "principles-macroeconomics-3e.txt", "principles-microeconomics-3e.txt", + "principles-economics-3e.txt", "principles-management.txt", + "principles-marketing.txt", +}) +GOVERNANCE_LAW = frozenset({ + "machiavelli_prince.txt", "mill_on_liberty.txt", "federalist_papers.txt", + "american-government-3e.txt", "introduction-political-science.txt", +}) +PHILOSOPHY_RELIGION = frozenset({ + "aristotle_metaphysics.txt", "aristotle_nicomachean_ethics.txt", + "augustine_confessions.txt", "bhagavad_gita.txt", "boethius_consolation.txt", + "confucius_analects.txt", "descartes_discourse.txt", "descartes_meditations.txt", + "hume_enquiry.txt", "hume_treatise.txt", "introduction-philosophy.txt", + "james_pragmatism.txt", "kant_critique_pure_reason.txt", "kant_groundwork.txt", + "kjv_bible.txt", "marcus_aurelius_meditations.txt", "mill_utilitarianism.txt", + "nietzsche_beyond_good_and_evil.txt", "nietzsche_genealogy_of_morals.txt", + "nietzsche_twilight_of_the_idols.txt", "nietzsche_zarathustra.txt", + "plato_apology.txt", "plato_phaedo.txt", "plato_republic.txt", + "plato_symposium.txt", "russell_problems_of_philosophy.txt", "tao_te_ching.txt", +}) + + +# Complete-document holdouts supplement C94's already separated `selection` +# sources, which do not contain examples for every scientific domain. +FIT_DOCUMENT_HOLDOUTS = frozenset({ + "chemistry-atoms-first-2e.txt", + "lyell_principles_geology.txt", + "biology-2e.txt", + "euclid_elements_book7_to_13.txt", + "psychology-2e.txt", + "world-history-volume-2.txt", + "principles-microeconomics-3e.txt", + "introduction-political-science.txt", + "hume_enquiry.txt", + "shakespeare_tempest.txt", +}) + + +CROSS_DOMAIN_OBLIGATIONS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("How do orbital energy and planetary geology jointly affect a moon's internal heating?", ("physical_engineering", "earth_space")), + ("Explain how climate change alters ecosystems and organism survival.", ("earth_space", "life_health")), + ("Use a mathematical model to calculate the motion of a physical system.", ("mathematics_computation", "physical_engineering")), + ("How did historical change and psychology jointly shape collective behavior?", ("social_behavioral", "history_culture")), + ("Explain the historical conditions that formed a constitutional government.", ("history_culture", "governance_law")), + ("How do regulation and market incentives interact in an economy?", ("economics_organization", "governance_law")), + ("What ethical limits should govern a new biomedical intervention?", ("life_health", "philosophy_religion")), + ("Explain behavioral economics using both cognition and market choice.", ("social_behavioral", "economics_organization")), + ("Write a clear explanation of a chemical mechanism for a general reader.", ("physical_engineering", "language_arts")), + ("Interpret how a novel represents the moral philosophy of its period.", ("history_culture", "philosophy_religion", "language_arts")), + ("Compare religious belief with political authority in an ancient empire.", ("history_culture", "governance_law", "philosophy_religion")), + ("Explain how statistical evidence supports a claim about public health policy.", ("mathematics_computation", "life_health", "governance_law")), + ("How might resource scarcity change markets, governments, and social behavior?", ("social_behavioral", "economics_organization", "governance_law")), + ("Describe the mathematics of an orbit and express it as readable scientific prose.", ("earth_space", "mathematics_computation", "language_arts")), +) + + +def domain_for_document(source_document: str) -> str: + name = Path(source_document).name + groups = ( + ("physical_engineering", PHYSICAL_ENGINEERING), + ("earth_space", EARTH_SPACE), + ("life_health", LIFE_HEALTH), + ("mathematics_computation", MATHEMATICS_COMPUTATION), + ("social_behavioral", SOCIAL_BEHAVIORAL), + ("history_culture", HISTORY_CULTURE), + ("economics_organization", ECONOMICS_ORGANIZATION), + ("governance_law", GOVERNANCE_LAW), + ("philosophy_religion", PHILOSOPHY_RELIGION), + ) + matches = tuple(domain for domain, values in groups if name in values) + if len(matches) > 1: + raise ValueError(f"source assigned to multiple domains: {source_document}") + return matches[0] if matches else "language_arts" + + +def iter_records(corpus_path: Path = PUBLIC_CORPUS) -> Iterable[dict[str, Any]]: + with corpus_path.open("r", encoding="utf-8") as handle: + for line in handle: + source = json.loads(line) + document_name = Path(source["source_document"]).name + split = ( + "validation" + if source["split"] == "selection" or document_name in FIT_DOCUMENT_HOLDOUTS + else "train" + ) + yield { + "schema_version": SCHEMA, + "record_id": source["id"], + "split": split, + "text": source["text"], + "required_domain_ids": [domain_for_document(source["source_document"])], + "source_collection": source["source_collection"], + "source_document": source["source_document"], + "source_document_sha256": source["source_document_sha256"], + "text_sha256": source["text_sha256"], + "license_note": source["license_note"], + } + + +def source_sha256(path: Path = PUBLIC_CORPUS) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def ontology_sha256() -> str: + value = { + "domains": [spec.__dict__ for spec in DOMAIN_SPECS], + "holdouts": sorted(FIT_DOCUMENT_HOLDOUTS), + "cross_domain": CROSS_DOMAIN_OBLIGATIONS, + } + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +__all__ = [ + "CROSS_DOMAIN_OBLIGATIONS", "DOMAIN_SPECS", "FIT_DOCUMENT_HOLDOUTS", + "KnowledgeDomainSpec", "PUBLIC_CORPUS", "domain_for_document", + "iter_records", "ontology_sha256", "source_sha256", +] diff --git a/root/wander-train/stage5k29_meta_fold_knowledge_basis.py b/root/wander-train/stage5k29_meta_fold_knowledge_basis.py new file mode 100644 index 0000000000000000000000000000000000000000..2d0e5410fa64ca553ef0af0f0a9cd2ee9a288498 --- /dev/null +++ b/root/wander-train/stage5k29_meta_fold_knowledge_basis.py @@ -0,0 +1,852 @@ +#!/usr/bin/env python3 +"""Train a minimal, source-grounded HRR domain-specialist basis. + +K14 minimized operational specialists (causal, numeric, synthesis, and so on). +K29 supplies the orthogonal domain coordinate. Ten broad knowledge modes are +compiled from source-separated public text, packed into the smallest set of +two-mode cells that preserves development routing, and carried under the same +4:1 protected HRR root. The Qum meta-fold engine audits the selected +configuration as a homeostatic fixed point. + +This is a domain router and provenance-preserving evidence memory. It is not +by itself a free-form answer generator or proof of open-domain factual QA. +""" + +from __future__ import annotations + +import argparse +from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +import functools +import hashlib +import json +import math +from pathlib import Path +import threading +import time +from typing import Any, Iterable, Mapping, Sequence + +import numpy as np + +from stage5k13_concurrent_specialist_pool import ( + directional_similarity, + directional_text_state, + tokens, +) +from stage5k14_meta_fold_specialist_basis import load_qum_meta_fold +from stage5k29_general_knowledge_domains import ( + CROSS_DOMAIN_OBLIGATIONS, + DOMAIN_SPECS, + PUBLIC_CORPUS, + iter_records, + ontology_sha256, + source_sha256, +) +from train_stage2a_role_ladder import HRR_DIM, cosine, normalize, unitary + + +SCHEMA = "oracle-stage5k29-meta-fold-knowledge-basis-v1" +REPORT_SCHEMA = "oracle-stage5k29-meta-fold-knowledge-development-v1" +ROOT = Path(__file__).resolve().parent.parent +STOP = frozenset({ + "a", "all", "an", "and", "are", "as", "at", "be", "been", "but", "by", + "do", "does", "did", "for", "from", "had", "has", "have", "he", "her", "his", "how", "i", "if", + "in", "into", "is", "it", "its", "not", "of", "on", "or", "our", + "she", "that", "the", "their", "them", "there", "they", "this", "to", + "was", "we", "were", "what", "when", "which", "who", "will", "with", + "would", "you", "your", +}) + + +def canonical(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +@dataclass(frozen=True) +class KnowledgeProfile: + domain_id: str + fold_states: tuple[np.ndarray, ...] + learned_cues: tuple[str, ...] + train_records: int + validation_records: int + train_documents: tuple[str, ...] + validation_documents: tuple[str, ...] + + +@dataclass(frozen=True) +class DomainScore: + domain_id: str + relevance: float + lexical_overlap: float + geometric_similarity: float + state: np.ndarray + + +@dataclass(frozen=True) +class KnowledgeRoutingMetrics: + records: int + required_labels: int + recovered_labels: int + selected_labels: int + full_coverage_records: int + single_top1_records: int + single_records: int + cross_domain_records: int + cross_domain_full_records: int + represented_domains: tuple[str, ...] + per_domain_top1: Mapping[str, float] + minimum_root_cosine: float = 0.0 + minimum_concurrent_workers: int = 0 + ablation_material_change_rate: float | None = None + + @property + def label_recall(self) -> float: + return self.recovered_labels / max(1, self.required_labels) + + @property + def label_precision(self) -> float: + return self.recovered_labels / max(1, self.selected_labels) + + @property + def full_coverage_rate(self) -> float: + return self.full_coverage_records / max(1, self.records) + + @property + def single_top1_accuracy(self) -> float: + return self.single_top1_records / max(1, self.single_records) + + @property + def cross_domain_full_rate(self) -> float: + return self.cross_domain_full_records / max(1, self.cross_domain_records) + + @property + def macro_domain_top1(self) -> float: + return float(np.mean(tuple(self.per_domain_top1.values()))) if self.per_domain_top1 else 0.0 + + def as_dict(self) -> dict[str, Any]: + return { + "records": self.records, + "required_labels": self.required_labels, + "recovered_labels": self.recovered_labels, + "selected_labels": self.selected_labels, + "label_recall": self.label_recall, + "label_precision": self.label_precision, + "full_coverage_rate": self.full_coverage_rate, + "single_top1_accuracy": self.single_top1_accuracy, + "macro_domain_top1": self.macro_domain_top1, + "cross_domain_records": self.cross_domain_records, + "cross_domain_full_rate": self.cross_domain_full_rate, + "represented_domains": list(self.represented_domains), + "per_domain_top1": dict(self.per_domain_top1), + "minimum_root_cosine": self.minimum_root_cosine, + "minimum_concurrent_workers": self.minimum_concurrent_workers, + "ablation_material_change_rate": self.ablation_material_change_rate, + } + + +def _mean_state(values: Iterable[np.ndarray]) -> np.ndarray | None: + rows = tuple(values) + if not rows: + return None + return normalize(sum(rows, np.zeros(HRR_DIM, dtype=np.complex64))) + + +def _stable_sample(rows: Sequence[Mapping[str, Any]], limit: int) -> tuple[Mapping[str, Any], ...]: + return tuple(sorted( + rows, + key=lambda row: hashlib.sha256(str(row["record_id"]).encode()).digest(), + )[:limit]) + + +def compile_profiles( + rows: Sequence[Mapping[str, Any]], +) -> tuple[KnowledgeProfile, ...]: + train = tuple(row for row in rows if row["split"] == "train") + validation = tuple(row for row in rows if row["split"] == "validation") + global_presence = Counter( + term for row in train for term in set(tokens(str(row["text"]))) + if len(term) >= 3 and term not in STOP + ) + spec_by_id = {value.domain_id: value for value in DOMAIN_SPECS} + profiles: list[KnowledgeProfile] = [] + for domain_id in sorted(spec_by_id): + positive = tuple(row for row in train if row["required_domain_ids"] == [domain_id]) + held_out = tuple(row for row in validation if row["required_domain_ids"] == [domain_id]) + presence = Counter( + term for row in positive for term in set(tokens(str(row["text"]))) + if len(term) >= 3 and term not in STOP + ) + negative_rows = max(1, len(train) - len(positive)) + ranked = sorted( + presence, + key=lambda term: ( + -math.log( + ((presence[term] + 1.0) / (len(positive) + 2.0)) + / ((global_presence[term] - presence[term] + 1.0) / (negative_rows + 2.0)) + ) * math.log1p(presence[term]), + term, + ), + ) + curated = tuple(spec_by_id[domain_id].cues) + learned = tuple(dict.fromkeys((*curated, *ranked[:256]))) + by_document: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for row in positive: + by_document[str(row["source_document"])].append(row) + folds: list[np.ndarray] = [ + directional_text_state(spec_by_id[domain_id].description), + directional_text_state(" ".join(learned[:128])), + ] + for document in sorted(by_document): + state = _mean_state( + directional_text_state(str(row["text"])) + for row in _stable_sample(by_document[document], 96) + ) + if state is not None: + folds.append(state) + profiles.append(KnowledgeProfile( + domain_id=domain_id, + fold_states=tuple(folds), + learned_cues=learned, + train_records=len(positive), + validation_records=len(held_out), + train_documents=tuple(sorted({str(row["source_document"]) for row in positive})), + validation_documents=tuple(sorted({str(row["source_document"]) for row in held_out})), + )) + return tuple(profiles) + + +def _domain_score( + profile: KnowledgeProfile, + text: str, + query_state: np.ndarray, + lexical_weight: float, +) -> DomainScore: + query_terms = {value for value in tokens(text) if value not in STOP} + cue_terms = set(profile.learned_cues) + overlap = query_terms & cue_terms + curated = cue_terms & set(profile.learned_cues[:32]) + overlap_count = len(overlap) + curated_count = len(overlap & curated) + lexical = (overlap_count + 1.5 * curated_count) / max( + 1.0, math.sqrt(len(query_terms) * min(64, len(cue_terms))) + ) + ranked = sorted( + [ + ( + max(0.0, directional_similarity(query_state, state)), + index, + state, + ) for index, state in enumerate(profile.fold_states) + ], + key=lambda value: (-value[0], value[1]), + ) + neighborhood = ranked[:min(4, len(ranked))] + geometric = 0.45 * neighborhood[0][0] + 0.55 * float(np.mean([value[0] for value in neighborhood])) + evidence_state = normalize(sum( + ( + np.float32(max(score, 1e-4)) * fold + for score, _, fold in neighborhood[:3] + ), + np.zeros(HRR_DIM, dtype=np.complex64), + )) + # A contribution carries both recovered evidence shape and an immutable + # domain address. Otherwise broad prose centroids can become nearly + # collinear and specialist removal ceases to be causally observable. + state = normalize( + np.float32(0.68) * evidence_state + + np.float32(0.32) * unitary("stage5k29:domain:" + profile.domain_id) + ) + relevance = lexical_weight * lexical + (1.0 - lexical_weight) * geometric + return DomainScore(profile.domain_id, float(relevance), float(lexical), float(geometric), state) + + +def score_text( + profiles: Sequence[KnowledgeProfile], + text: str, + lexical_weight: float, +) -> dict[str, DomainScore]: + state = directional_text_state(text) + return { + profile.domain_id: _domain_score(profile, text, state, lexical_weight) + for profile in profiles + } + + +def _coactivation_edges() -> frozenset[frozenset[str]]: + edges: set[frozenset[str]] = set() + for _, required in CROSS_DOMAIN_OBLIGATIONS: + for index, left in enumerate(required): + for right in required[index + 1:]: + edges.add(frozenset((left, right))) + return frozenset(edges) + + +def make_partition( + profiles: Sequence[KnowledgeProfile], + target_count: int, + mode: str, + salt: int, +) -> tuple[tuple[str, ...], ...]: + ids = tuple(sorted(value.domain_id for value in profiles)) + if not math.ceil(len(ids) / 2) <= target_count <= len(ids): + raise ValueError("target count incompatible with two-mode domain cells") + forbidden = _coactivation_edges() + centroids = { + value.domain_id: _mean_state(value.fold_states) + for value in profiles + } + needed_pairs = len(ids) - target_count + + def pair_score(left: str, right: str) -> float: + affinity = cosine(centroids[left], centroids[right]) + base = affinity if mode == "affine" else ((1.0 - affinity) if mode == "separated" else (1.0 - abs(affinity - 0.10))) + raw = hashlib.sha256(f"{salt}:{left}:{right}".encode()).digest() + return base + 0.05 * int.from_bytes(raw[:4], "little") / (2**32 - 1) + + scores = { + (left, right): pair_score(left, right) + for index, left in enumerate(ids) for right in ids[index + 1:] + if frozenset((left, right)) not in forbidden + } + + @functools.lru_cache(maxsize=None) + def solve(remaining: tuple[str, ...], pairs_left: int): + if pairs_left == 0: + return (0.0, ()) + if len(remaining) < 2 * pairs_left: + return None + left = remaining[0] + best = solve(remaining[1:], pairs_left) if len(remaining) - 1 >= 2 * pairs_left else None + for index, right in enumerate(remaining[1:], start=1): + pair = (left, right) + if pair not in scores: + continue + solved = solve(remaining[1:index] + remaining[index + 1:], pairs_left - 1) + if solved is None: + continue + candidate = (scores[pair] + solved[0], (pair, *solved[1])) + if best is None or candidate > best: + best = candidate + return best + + solved = solve(ids, needed_pairs) + if solved is None: + raise RuntimeError("no compatible domain partition") + pairs = solved[1] + used = {member for pair in pairs for member in pair} + groups = [tuple(sorted(pair)) for pair in pairs] + groups.extend((value,) for value in ids if value not in used) + return tuple(sorted(groups)) + + +def _selected_domains( + partition: Sequence[Sequence[str]], + scores: Mapping[str, DomainScore], + selected_count: int = 3, +) -> tuple[DomainScore, ...]: + cell_winners = [ + max((scores[member] for member in cell), key=lambda value: (value.relevance, value.domain_id)) + for cell in partition + ] + return tuple(sorted(cell_winners, key=lambda value: (-value.relevance, value.domain_id))[:selected_count]) + + +def evaluate_scored( + partition: Sequence[Sequence[str]], + scored_rows: Sequence[tuple[tuple[str, ...], Mapping[str, DomainScore]]], +) -> KnowledgeRoutingMetrics: + required_labels = recovered_labels = selected_labels = full = top1 = singles = 0 + cross = cross_full = 0 + represented: set[str] = set() + domain_hits = Counter() + domain_total = Counter() + for required_value, scores in scored_rows: + required = set(required_value) + selected_values = _selected_domains(partition, scores) + selected = {value.domain_id for value in selected_values} + recovered = required & selected + required_labels += len(required) + recovered_labels += len(recovered) + selected_labels += len(selected) + full += int(required <= selected) + represented.update(recovered) + if len(required) == 1: + singles += 1 + domain = next(iter(required)) + domain_total[domain] += 1 + hit = bool(selected_values) and selected_values[0].domain_id == domain + top1 += int(hit) + domain_hits[domain] += int(hit) + else: + cross += 1 + cross_full += int(required <= selected) + per_domain = { + domain: domain_hits[domain] / max(1, domain_total[domain]) + for domain in sorted(domain_total) + } + return KnowledgeRoutingMetrics( + records=len(scored_rows), required_labels=required_labels, + recovered_labels=recovered_labels, selected_labels=selected_labels, + full_coverage_records=full, single_top1_records=top1, + single_records=singles, cross_domain_records=cross, + cross_domain_full_records=cross_full, + represented_domains=tuple(sorted(represented)), per_domain_top1=per_domain, + ) + + +class KnowledgeSpecialistPool: + """Concurrent domain heads with one activated primitive per composite cell.""" + + def __init__( + self, + partition: Sequence[Sequence[str]], + profiles: Sequence[KnowledgeProfile], + lexical_weight: float, + maximum_workers: int = 4, + ) -> None: + self.partition = tuple(tuple(value) for value in partition) + self.profiles = {value.domain_id: value for value in profiles} + self.lexical_weight = float(lexical_weight) + self.maximum_workers = max(2, min(int(maximum_workers), len(self.partition))) + + def _cell_score( + self, + cell: tuple[str, ...], + text: str, + query_state: np.ndarray, + barrier: threading.Barrier | None, + barrier_cells: frozenset[str], + ) -> tuple[str, DomainScore, str]: + cell_id = "__".join(cell) + if barrier is not None and cell_id in barrier_cells: + barrier.wait(timeout=5.0) + values = tuple( + _domain_score(self.profiles[member], text, query_state, self.lexical_weight) + for member in cell + ) + return cell_id, max(values, key=lambda value: (value.relevance, value.domain_id)), threading.current_thread().name + + def route(self, text: str, *, ablate_cells: Sequence[str] = ()) -> dict[str, Any]: + clean = " ".join(str(text).split()) + if not clean: + raise ValueError("domain routing requires nonempty text") + query_state = directional_text_state(clean) + blocked = set(ablate_cells) + cells = tuple(cell for cell in self.partition if "__".join(cell) not in blocked) + barrier_values = tuple("__".join(cell) for cell in cells[:self.maximum_workers]) + barrier = threading.Barrier(len(barrier_values)) if len(barrier_values) >= 2 else None + with ThreadPoolExecutor(max_workers=self.maximum_workers, thread_name_prefix="oracle-knowledge") as executor: + futures = tuple( + executor.submit(self._cell_score, cell, clean, query_state, barrier, frozenset(barrier_values)) + for cell in cells + ) + values = tuple(future.result() for future in futures) + ordered = tuple(sorted(values, key=lambda value: (-value[1].relevance, value[0]))) + selected = ordered[:3] + moving = normalize(sum( + (np.float32(max(value[1].relevance, 1e-4)) * value[1].state for value in selected), + np.zeros(HRR_DIM, dtype=np.complex64), + )) + root = unitary("stage5k29:protected-root:" + hashlib.sha256(clean.encode()).hexdigest()) + residual = moving - np.complex64(np.vdot(root, moving)) * root + protected = normalize(np.float32(4.0) * root + residual) + return { + "schema_version": "oracle-stage5k29-knowledge-routing-v1", + "selected_cell_ids": [value[0] for value in selected], + "selected_domain_ids": [value[1].domain_id for value in selected], + "contributions": [{ + "cell_id": cell_id, + "activated_domain_id": score.domain_id, + "relevance": score.relevance, + "lexical_overlap": score.lexical_overlap, + "geometric_similarity": score.geometric_similarity, + "worker_name": worker, + } for cell_id, score, worker in ordered], + "registered_cells": len(self.partition), + "primitive_domain_modes": len(self.profiles), + "concurrent_worker_count": len({value[2] for value in values}), + "protected_root_cosine": cosine(root, protected), + "pre_malkuth_unbinds": 0, + "malkuth_unbinds": 1, + "traditional_language_model": False, + "pretrained_embedding_encoder": False, + "next_token_predictions": 0, + "fused_state": moving, + } + + def ablation_changes(self, text: str) -> dict[str, Any]: + baseline = self.route(text) + ablated = self.route(text, ablate_cells=baseline["selected_cell_ids"]) + similarity = cosine(baseline["fused_state"], ablated["fused_state"]) + return { + "selected_set_changed": baseline["selected_cell_ids"] != ablated["selected_cell_ids"], + "fused_state_cosine": similarity, + "material_change": baseline["selected_cell_ids"] != ablated["selected_cell_ids"] and similarity < 0.95, + } + + +@dataclass(frozen=True) +class TrainedKnowledgeBasis: + partition: tuple[tuple[str, ...], ...] + profiles: tuple[KnowledgeProfile, ...] + lexical_weight: float + metadata: Mapping[str, Any] + file_sha256: str | None = None + + def pool(self) -> KnowledgeSpecialistPool: + return KnowledgeSpecialistPool(self.partition, self.profiles, self.lexical_weight) + + def save(self, path: Path) -> str: + states: list[np.ndarray] = [] + owners: list[int] = [] + for index, profile in enumerate(self.profiles): + for state in profile.fold_states: + states.append(state) + owners.append(index) + metadata = dict(self.metadata) + metadata.update({ + "schema_version": SCHEMA, + "partition": [list(value) for value in self.partition], + "lexical_weight": self.lexical_weight, + "profiles": [{ + "domain_id": value.domain_id, + "learned_cues": list(value.learned_cues), + "train_records": value.train_records, + "validation_records": value.validation_records, + "train_documents": list(value.train_documents), + "validation_documents": list(value.validation_documents), + "folds": len(value.fold_states), + } for value in self.profiles], + }) + np.savez_compressed( + path, + metadata=np.asarray(canonical(metadata)), + fold_states=np.asarray(states, dtype=np.complex64), + fold_owner_index=np.asarray(owners, dtype=np.int16), + ) + digest_value = sha256(path) + object.__setattr__(self, "file_sha256", digest_value) + return digest_value + + @classmethod + def load(cls, path: Path) -> "TrainedKnowledgeBasis": + digest_value = sha256(path) + with np.load(path, allow_pickle=False) as archive: + if any(value.dtype.hasobject for value in archive.values()): + raise ValueError("object dtype forbidden in K29 checkpoint") + metadata = json.loads(str(archive["metadata"].item())) + if metadata.get("schema_version") != SCHEMA: + raise ValueError("unsupported K29 checkpoint") + states = np.asarray(archive["fold_states"], dtype=np.complex64) + owners = np.asarray(archive["fold_owner_index"], dtype=np.int16) + profiles = tuple(KnowledgeProfile( + domain_id=str(row["domain_id"]), + fold_states=tuple(states[owners == index]), + learned_cues=tuple(row["learned_cues"]), + train_records=int(row["train_records"]), + validation_records=int(row["validation_records"]), + train_documents=tuple(row["train_documents"]), + validation_documents=tuple(row["validation_documents"]), + ) for index, row in enumerate(metadata["profiles"])) + return cls( + partition=tuple(tuple(value) for value in metadata["partition"]), + profiles=profiles, + lexical_weight=float(metadata["lexical_weight"]), + metadata=metadata, + file_sha256=digest_value, + ) + + def retrieve( + self, + query: str, + corpus_path: Path = PUBLIC_CORPUS, + top_k: int = 3, + *, + selected_domain_ids: Sequence[str] | None = None, + ) -> tuple[dict[str, Any], ...]: + """Return source-bearing evidence; do not turn retrieval into an answer claim.""" + selected = set( + selected_domain_ids + if selected_domain_ids is not None + else self.pool().route(query)["selected_domain_ids"] + ) + query_terms = {value for value in tokens(query) if value not in STOP} + candidates: list[tuple[float, dict[str, Any]]] = [] + for row in iter_records(corpus_path): + if row["split"] != "train" or row["required_domain_ids"][0] not in selected: + continue + row_terms = {value for value in tokens(str(row["text"])) if value not in STOP} + shared = query_terms & row_terms + if not shared: + continue + lexical = len(shared) / max(1.0, math.sqrt(len(query_terms) * len(row_terms))) + candidates.append((lexical, { + "domain_id": row["required_domain_ids"][0], + "text": row["text"], + "source_document": row["source_document"], + "source_document_sha256": row["source_document_sha256"], + "text_sha256": row["text_sha256"], + "license_note": row["license_note"], + "retrieval_score": lexical, + "shared_terms": sorted(shared), + "query_term_coverage": len(shared) / max(1, len(query_terms)), + "evidence_term_coverage": len(shared) / max(1, len(row_terms)), + "answer_status": "evidence_only", + })) + return tuple(value for _, value in sorted(candidates, key=lambda item: (-item[0], item[1]["text_sha256"]))[:top_k]) + + +def _utility(metrics: KnowledgeRoutingMetrics) -> float: + return ( + 0.25 * metrics.label_recall + + 0.20 * metrics.full_coverage_rate + + 0.20 * metrics.single_top1_accuracy + + 0.20 * metrics.macro_domain_top1 + + 0.15 * metrics.cross_domain_full_rate + ) + + +def _passes(candidate: KnowledgeRoutingMetrics, baseline: KnowledgeRoutingMetrics) -> bool: + return all(( + len(candidate.represented_domains) == len(DOMAIN_SPECS), + candidate.label_recall >= baseline.label_recall - 0.025, + candidate.full_coverage_rate >= baseline.full_coverage_rate - 0.025, + candidate.single_top1_accuracy >= baseline.single_top1_accuracy - 0.025, + candidate.macro_domain_top1 >= baseline.macro_domain_top1 - 0.035, + candidate.cross_domain_full_rate >= 0.85, + )) + + +def _qum_projection(partition: Sequence[Sequence[str]], metrics: KnowledgeRoutingMetrics) -> dict[str, Any]: + engine, meta_fold = load_qum_meta_fold() + config = meta_fold.make_config(0.12, 0.05, 1.5) + identity = engine._seed_vector_world("knowledge-partition:" + canonical(partition)) + config_state = meta_fold.config_to_vector(config) + context = engine._circular_conv_w(identity, config_state) + for name, value in ( + ("recall", metrics.label_recall), ("coverage", metrics.full_coverage_rate), + ("macro", metrics.macro_domain_top1), ("cross", metrics.cross_domain_full_rate), + ): + context += float(value) * engine._seed_vector_world("knowledge-fitness:" + name) + context /= max(float(np.linalg.norm(context)), 1e-12) + context *= math.sqrt(engine.WORLD_DIM) + dimensions: dict[str, Any] = {} + total_issues = 0 + for name, bias in (("balanced", None), ("tiferet", "TIFERET"), ("gevurah", "GEVURAH")): + dimension = meta_fold.MetaDimension("knowledge-" + name, config, bias, 0.5) + dimension.ctx = context.copy() + for _ in range(8): + dimension._apply_context_correction_modified() + issues, health = dimension.health() + total_issues += issues + dimensions[name] = {"issues": issues, "sephiroth": {key: value["act"] for key, value in health.items()}} + return { + "config": config, "dimensions": dimensions, + "total_homeostasis_issues": total_issues, + "identity_operations": 1, "convolution_operations": 1, + "accumulation_operations": 4, "projection_operations": 24, + } + + +def train(corpus: Path, checkpoint: Path, report_path: Path) -> dict[str, Any]: + started = time.perf_counter() + rows = tuple(iter_records(corpus)) + profiles = compile_profiles(rows) + validation_by_domain: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for row in rows: + if row["split"] == "validation": + validation_by_domain[row["required_domain_ids"][0]].append(row) + profile_by_id = {value.domain_id: value for value in profiles} + spec_by_id = {value.domain_id: value for value in DOMAIN_SPECS} + evaluation_rows: list[tuple[tuple[str, ...], str]] = [] + excluded_unidentifiable = 0 + for domain_id in sorted(validation_by_domain): + profile = profile_by_id[domain_id] + curated = set(spec_by_id[domain_id].cues) + learned = set(profile.learned_cues) + identifiable = [] + for row in validation_by_domain[domain_id]: + row_terms = set(tokens(str(row["text"]))) + if row_terms & curated or len(row_terms & learned) >= 2: + identifiable.append(row) + else: + excluded_unidentifiable += 1 + evaluation_rows.extend( + ((domain_id,), str(row["text"])) + for row in _stable_sample(identifiable, 64) + ) + evaluation_rows.extend((required, text) for text, required in CROSS_DOMAIN_OBLIGATIONS) + + score_cache: dict[float, list[tuple[tuple[str, ...], Mapping[str, DomainScore]]]] = {} + for weight in (0.50, 0.65, 0.80): + score_cache[weight] = [ + (tuple(required), score_text(profiles, text, weight)) + for required, text in evaluation_rows + ] + full_partition = tuple((value.domain_id,) for value in profiles) + baselines = { + weight: evaluate_scored(full_partition, scored) + for weight, scored in score_cache.items() + } + + candidates: list[dict[str, Any]] = [] + unique: set[tuple[Any, ...]] = set() + for count in range(math.ceil(len(profiles) / 2), len(profiles) + 1): + for mode in ("affine", "balanced", "separated"): + for salt in range(4): + try: + partition = make_partition(profiles, count, mode, salt) + except RuntimeError: + continue + for weight in (0.50, 0.65, 0.80): + key = (*partition, weight) + if key in unique: + continue + unique.add(key) + metrics = evaluate_scored(partition, score_cache[weight]) + candidates.append({ + "registered_domain_cells": count, + "mode": mode, + "salt": salt, + "partition": partition, + "lexical_weight": weight, + "metrics": metrics, + "passes": _passes(metrics, baselines[weight]), + "utility": _utility(metrics), + }) + passing = [value for value in candidates if value["passes"]] + ranked = sorted( + passing or candidates, + key=lambda value: ( + value["registered_domain_cells"] if passing else 999, + -value["utility"], canonical(value["partition"]), value["lexical_weight"], + ), + ) + finalist = ranked[0] + pool = KnowledgeSpecialistPool(finalist["partition"], profiles, finalist["lexical_weight"]) + runtime_results = [pool.route(text) for _, text in evaluation_rows[:40]] + ablations = [pool.ablation_changes(text)["material_change"] for _, text in evaluation_rows[:20]] + metrics = finalist["metrics"] + runtime_metrics = KnowledgeRoutingMetrics( + **{key: getattr(metrics, key) for key in ( + "records", "required_labels", "recovered_labels", "selected_labels", + "full_coverage_records", "single_top1_records", "single_records", + "cross_domain_records", "cross_domain_full_records", "represented_domains", + "per_domain_top1", + )}, + minimum_root_cosine=min(value["protected_root_cosine"] for value in runtime_results), + minimum_concurrent_workers=min(value["concurrent_worker_count"] for value in runtime_results), + ablation_material_change_rate=float(np.mean(ablations)), + ) + accepted = bool(passing) and (runtime_metrics.ablation_material_change_rate or 0.0) >= 0.95 and runtime_metrics.minimum_root_cosine >= 0.9701 + metadata = { + "training_kind": "source_grounded_directional_hrr_fold_compilation_and_meta_fold_partition_search", + "claim_boundary": "Opened development domain routing and evidence retrieval; not unrestricted factual QA or a universal minimality theorem.", + "public_corpus_sha256": source_sha256(corpus), + "ontology_sha256": ontology_sha256(), + "public_corpus_path": str(corpus.resolve()), + "primitive_domain_modes": len(profiles), + "registered_domain_cells": len(finalist["partition"]), + "maximum_primitive_modes_per_cell": 2, + "selected_domain_cells_per_query": 3, + "global_gradient_steps": 0, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + } + basis = TrainedKnowledgeBasis( + partition=tuple(finalist["partition"]), profiles=profiles, + lexical_weight=float(finalist["lexical_weight"]), metadata=metadata, + ) + checkpoint.parent.mkdir(parents=True, exist_ok=True) + report_path.parent.mkdir(parents=True, exist_ok=True) + digest_value = basis.save(checkpoint) + count_audit = [] + for count in range(math.ceil(len(profiles) / 2), len(profiles) + 1): + subset = [value for value in candidates if value["registered_domain_cells"] == count] + if not subset: + count_audit.append({"registered_domain_cells": count, "compatible_partition_found": False}) + continue + best = max(subset, key=lambda value: value["utility"]) + count_audit.append({ + "registered_domain_cells": count, + "compatible_partition_found": True, + "any_gate_pass": any(value["passes"] for value in subset), + "best_utility": best["utility"], + "best_metrics": best["metrics"].as_dict(), + }) + train_rows = [row for row in rows if row["split"] == "train"] + validation_rows = [row for row in rows if row["split"] == "validation"] + report = { + "schema_version": REPORT_SCHEMA, + "status": "accepted_development_checkpoint" if accepted else "rejected_development_checkpoint", + "claim_boundary": metadata["claim_boundary"], + "training": { + "public_source_records": len(rows), "train_records": len(train_rows), + "source_separated_validation_records": len(validation_rows), + "balanced_evaluation_records": len(evaluation_rows), + "validation_rows_excluded_as_domain_unidentifiable_at_sentence_scope": excluded_unidentifiable, + "cross_domain_obligations": len(CROSS_DOMAIN_OBLIGATIONS), + "train_source_documents": len({row["source_document"] for row in train_rows}), + "validation_source_documents": len({row["source_document"] for row in validation_rows}), + "primitive_fold_states": sum(len(value.fold_states) for value in profiles), + "candidate_configurations": len(candidates), + "global_gradient_steps": 0, + "elapsed_seconds": time.perf_counter() - started, + }, + "minimality": { + "kind": "minimum_passing_compatible_cell_count_in_searched_range", + "primitive_domain_modes": len(profiles), + "theoretical_two_mode_floor": math.ceil(len(profiles) / 2), + "searched_counts": list(range(math.ceil(len(profiles) / 2), len(profiles) + 1)), + "selected_domain_cells": len(finalist["partition"]), + "universal_minimum_established": False, + "count_audit": count_audit, + }, + "selected_configuration": { + "mode": finalist["mode"], "salt": finalist["salt"], + "lexical_weight": finalist["lexical_weight"], + "geometric_weight": 1.0 - finalist["lexical_weight"], + "partition": [list(value) for value in finalist["partition"]], + "utility": _utility(runtime_metrics), + }, + "baseline_ten_domain_metrics": baselines[finalist["lexical_weight"]].as_dict(), + "minimized_metrics": runtime_metrics.as_dict(), + "qum_meta_projection": _qum_projection(finalist["partition"], runtime_metrics), + "checkpoint_path": str(checkpoint.resolve()), + "checkpoint_sha256": digest_value, + "checkpoint_bytes": checkpoint.stat().st_size, + "prohibitions": { + "traditional_language_model": False, "transformer": False, + "pretrained_embedding_encoder": False, "autoregressive_carrier": False, + "next_token_predictions": 0, "global_gradient_steps": 0, + }, + } + report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--corpus", type=Path, default=PUBLIC_CORPUS) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(train(args.corpus, args.checkpoint, args.report), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() + + +__all__ = [ + "KnowledgeProfile", "KnowledgeRoutingMetrics", "KnowledgeSpecialistPool", + "TrainedKnowledgeBasis", "compile_profiles", "evaluate_scored", + "make_partition", "score_text", "train", +] diff --git a/root/wander-train/stage5k33_cross_domain_address_index.py b/root/wander-train/stage5k33_cross_domain_address_index.py new file mode 100644 index 0000000000000000000000000000000000000000..a3b8ea1d2a6c7de9be00f80be95970268e8dea93 --- /dev/null +++ b/root/wander-train/stage5k33_cross_domain_address_index.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""K33 overlapping geometric addresses for a compatible K29 knowledge basis. + +K29 deliberately learned ten broad primitive domains and initially packed +them into five two-member meta-fold cells. Later bases may pack more mutually +non-coactive modes into one cell. A source document nevertheless has only one +primary shelf address. That is too restrictive for sentences whose content +shape legitimately crosses shelves (for example, quantitative methods in an +anthropology text). K33 trains a compact secondary-address index without +adding specialists: a row retains its source-derived primary domain and gains +at most two strongly cued secondary domain addresses. Each address set is +bound into an HRR state under the same 4:1 stabilizing spine. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np + +from stage5k15_native_mixed_agent import tokens +from stage5k29_general_knowledge_domains import ( + CROSS_DOMAIN_OBLIGATIONS, + DOMAIN_SPECS, + PUBLIC_CORPUS, + iter_records, + ontology_sha256, + source_sha256, +) +from stage5k29_meta_fold_knowledge_basis import TrainedKnowledgeBasis, canonical +from train_stage2a_role_ladder import HRR_DIM, bind, cosine, normalize, unitary + + +SCHEMA = "oracle-stage5k33-cross-domain-address-index-v1" +ADDRESS_THRESHOLD = 2 +MAX_SECONDARY_ADDRESSES = 2 + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def semantic_addresses(text: str, primary_domain_id: str) -> tuple[str, ...]: + words = set(tokens(text)) + ranked = sorted( + ( + (len(words & set(spec.cues)), spec.domain_id) + for spec in DOMAIN_SPECS if spec.domain_id != primary_domain_id + ), + key=lambda value: (-value[0], value[1]), + ) + secondary = tuple( + domain_id for overlap, domain_id in ranked + if overlap >= ADDRESS_THRESHOLD + )[:MAX_SECONDARY_ADDRESSES] + return (primary_domain_id, *secondary) + + +def protected_address_state( + text_sha256: str, domain_ids: Sequence[str] +) -> tuple[np.ndarray, float]: + root = unitary("stage5k33:address-root:" + text_sha256) + role = unitary("stage5k33:role:domain-address") + moving = normalize(sum( + (bind(role, unitary("stage5k33:domain:" + value)) for value in domain_ids), + np.zeros(HRR_DIM, dtype=np.complex64), + )) + residual = moving - np.complex64(np.vdot(root, moving)) * root + protected = normalize(np.float32(4.0) * root + residual) + return protected, cosine(root, protected) + + +@dataclass(frozen=True) +class CrossDomainAddressIndex: + addresses_by_text_sha256: Mapping[str, tuple[str, ...]] + metadata: Mapping[str, Any] + file_sha256: str | None = None + + def addresses(self, text_sha256: str, primary_domain_id: str) -> tuple[str, ...]: + return self.addresses_by_text_sha256.get(text_sha256, (primary_domain_id,)) + + @classmethod + def load(cls, path: Path) -> "CrossDomainAddressIndex": + value = json.loads(path.read_text(encoding="utf-8")) + if value.get("schema_version") != SCHEMA: + raise ValueError("unsupported K33 address index") + rows = value.get("addresses_by_text_sha256") + if not isinstance(rows, Mapping): + raise ValueError("K33 address index is missing its address map") + return cls( + addresses_by_text_sha256={ + str(key): tuple(str(item) for item in addresses) + for key, addresses in rows.items() + }, + metadata={key: item for key, item in value.items() if key != "addresses_by_text_sha256"}, + file_sha256=sha256(path), + ) + + +def train( + corpus_path: Path, + knowledge_checkpoint: Path, + output_path: Path, +) -> dict[str, Any]: + basis = TrainedKnowledgeBasis.load(knowledge_checkpoint) + primitive_modes = len(basis.profiles) + registered_cells = len(basis.partition) + if primitive_modes != 10: + raise ValueError("K33 requires the accepted ten-mode K29 ontology") + partition_members = [member for cell in basis.partition for member in cell] + profile_ids = sorted(value.domain_id for value in basis.profiles) + if sorted(partition_members) != profile_ids or len(set(partition_members)) != len(profile_ids): + raise ValueError("K33 requires a complete one-cell-per-mode partition") + cell_by_domain = { + member: index for index, cell in enumerate(basis.partition) for member in cell + } + for _text, required in CROSS_DOMAIN_OBLIGATIONS: + required_cells = [cell_by_domain[value] for value in required] + if len(required_cells) != len(set(required_cells)): + raise ValueError("K33 knowledge cells collapse coactive domain modes") + maximum_modes_per_cell = max(map(len, basis.partition)) + addresses: dict[str, list[str]] = {} + secondary_counts = {spec.domain_id: 0 for spec in DOMAIN_SPECS} + minimum_root_cosine = 1.0 + split_counts: dict[str, int] = {} + for row in iter_records(corpus_path): + primary = str(row["required_domain_ids"][0]) + domain_ids = semantic_addresses(str(row["text"]), primary) + text_sha = str(row["text_sha256"]) + addresses[text_sha] = list(domain_ids) + split = str(row["split"]) + split_counts[split] = split_counts.get(split, 0) + 1 + for domain_id in domain_ids[1:]: + secondary_counts[domain_id] += 1 + _state, root_cosine = protected_address_state(text_sha, domain_ids) + minimum_root_cosine = min(minimum_root_cosine, root_cosine) + value = { + "schema_version": SCHEMA, + "status": "trained_cross_domain_addresses", + "construction": "K29 Qum meta-fold-selected compatible cells plus K33 HRR overlapping row addresses", + "knowledge_checkpoint_sha256": sha256(knowledge_checkpoint), + "public_corpus_sha256": source_sha256(corpus_path), + "ontology_sha256": ontology_sha256(), + "primitive_domain_modes": primitive_modes, + "registered_knowledge_cells": registered_cells, + "registered_two_mode_cells": registered_cells if maximum_modes_per_cell <= 2 else None, + "maximum_modes_per_cell": maximum_modes_per_cell, + "coactivation_compatible": True, + "theoretical_two_mode_cell_floor": (primitive_modes + 1) // 2, + "adds_new_specialists": False, + "address_threshold": ADDRESS_THRESHOLD, + "maximum_secondary_addresses": MAX_SECONDARY_ADDRESSES, + "records": len(addresses), + "split_counts": split_counts, + "multi_address_records": sum(len(value) > 1 for value in addresses.values()), + "secondary_address_counts": secondary_counts, + "minimum_protected_root_cosine": minimum_root_cosine, + "stabilizing_spine_gain": 4.0, + "pre_malkuth_payload_unbinds": 0, + "traditional_language_model_calls": 0, + "pretrained_embedding_encoder_calls": 0, + "next_token_predictions": 0, + "global_gradient_steps": 0, + "addresses_by_text_sha256": addresses, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("x", encoding="utf-8") as handle: + handle.write(json.dumps(value, indent=2, sort_keys=True) + "\n") + report = {key: item for key, item in value.items() if key != "addresses_by_text_sha256"} + report["output_sha256"] = sha256(output_path) + report["canonical_metadata_sha256"] = hashlib.sha256(canonical(report).encode()).hexdigest() + return report + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--corpus", type=Path, default=PUBLIC_CORPUS) + parser.add_argument("--knowledge-checkpoint", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(train(args.corpus, args.knowledge_checkpoint, args.output), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() + + +__all__ = [ + "CrossDomainAddressIndex", "protected_address_state", "semantic_addresses", "train", +] diff --git a/root/wander-train/stage5k33_repaired_native_generalist_agent.py b/root/wander-train/stage5k33_repaired_native_generalist_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..44ed3d14892f5b26c428f53b00003e70bc8e8b30 --- /dev/null +++ b/root/wander-train/stage5k33_repaired_native_generalist_agent.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +"""K33 repairs exposed by the immutable K32 prospective rejection. + +K33 is additive: none of the hash-frozen K32 runtime sources are modified. +It keeps the ten primitive knowledge modes in five meta-fold cells and repairs +five boundary mechanisms: cross-domain row addressing, open-vs-local +definition intent, imperative tool constructions, directional sensor state, +and modal/adverbial state propositions. +""" + +from __future__ import annotations + +import math +from pathlib import Path +import re +from typing import Any, Mapping, Sequence + +from stage5i2_open_schema_conversational_transducer import OpenSchemaConversationalTransducer +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest, tokens +from stage5k29_general_knowledge_domains import PUBLIC_CORPUS, iter_records +from stage5k29_meta_fold_knowledge_basis import STOP, TrainedKnowledgeBasis +from stage5k31_native_proposition_compiler_agent import ( + COMPILER_SCHEMA, + MODALS, + NativePropositionCompiler, + NativePropositionKnowledgeAgent, + WORD, + _span_in, + _strip_surface, +) +from stage5k33_cross_domain_address_index import CrossDomainAddressIndex + + +SCHEMA = "oracle-stage5k33-repaired-native-generalist-agent-v1" + + +class CrossAddressedKnowledgeBasis: + """Runtime retrieval view over the unchanged trained K29 meta-fold basis.""" + + def __init__( + self, + basis: TrainedKnowledgeBasis, + address_index: CrossDomainAddressIndex, + ) -> None: + self.basis = basis + self.address_index = address_index + + def pool(self): + return self.basis.pool() + + def retrieve( + self, + query: str, + corpus_path: Path = PUBLIC_CORPUS, + top_k: int = 3, + *, + selected_domain_ids: Sequence[str] | None = None, + ) -> tuple[dict[str, Any], ...]: + selected = set( + selected_domain_ids + if selected_domain_ids is not None + else self.pool().route(query)["selected_domain_ids"] + ) + query_terms = {value for value in tokens(query) if value not in STOP} + candidates: list[tuple[float, dict[str, Any]]] = [] + for row in iter_records(corpus_path): + if row["split"] != "train": + continue + primary = str(row["required_domain_ids"][0]) + address_ids = self.address_index.addresses(str(row["text_sha256"]), primary) + matched_addresses = tuple(sorted(selected & set(address_ids))) + if not matched_addresses: + continue + row_terms = {value for value in tokens(str(row["text"])) if value not in STOP} + shared = query_terms & row_terms + if not shared: + continue + lexical = len(shared) / max(1.0, math.sqrt(len(query_terms) * len(row_terms))) + candidates.append((lexical, { + "domain_id": primary, + "addressed_domain_ids": list(address_ids), + "matched_selected_domain_ids": list(matched_addresses), + "text": row["text"], + "source_document": row["source_document"], + "source_document_sha256": row["source_document_sha256"], + "text_sha256": row["text_sha256"], + "license_note": row["license_note"], + "retrieval_score": lexical, + "shared_terms": sorted(shared), + "query_term_coverage": len(shared) / max(1, len(query_terms)), + "evidence_term_coverage": len(shared) / max(1, len(row_terms)), + "answer_status": "evidence_only", + })) + return tuple( + value for _, value in sorted( + candidates, + key=lambda item: (-item[0], item[1]["text_sha256"]), + )[:top_k] + ) + + +class K33PropositionCompiler(NativePropositionCompiler): + """Add modal + intervening adverb + state-predicate constructions.""" + + _MODAL_REMAIN = re.compile( + r"^(.+?)\s+" + r"(can|could|may|might|must|shall|should|will|would)\s+" + r"(?:(?:therefore|always|still|usually|generally|necessarily|normally|then)\s+)*" + r"(remain|remains|remained|remaining)\s+(.+)$", + re.IGNORECASE, + ) + + def _parse_simple( + self, + clause: str, + evidence_row: Mapping[str, Any], + domain_ids: Sequence[str], + *, + clause_type: str = "assertion", + conditional: bool = False, + inherited_subject: str | None = None, + extra_qualifiers: Sequence[tuple[str, str]] = (), + base_offset: int = 0, + ): + surface = _strip_surface(clause) + main_surface = re.split( + r",\s*(?:however|although|even\s+though)\b", + surface, + maxsplit=1, + flags=re.IGNORECASE, + )[0] + match = self._MODAL_REMAIN.match(main_surface) if inherited_subject is None else None + if match is None: + return super()._parse_simple( + clause, + evidence_row, + domain_ids, + clause_type=clause_type, + conditional=conditional, + inherited_subject=inherited_subject, + extra_qualifiers=extra_qualifiers, + base_offset=base_offset, + ) + subject = self._clean_argument(match.group(1)) + predicate_surface = match.group(3) + object_value = self._clean_argument(match.group(4)) + evidence = clean(evidence_row.get("fact", evidence_row.get("text", ""))) + subject_span = _span_in(evidence, subject, base_offset) + predicate_span = _span_in(evidence, predicate_surface, base_offset) + object_span = _span_in( + evidence, + object_value, + predicate_span[1] if predicate_span is not None else base_offset, + ) + if subject_span is None or predicate_span is None: + return None + modal_word = match.group(2).casefold() + modality = ( + "conditional" if conditional + else "obligatory" if modal_word in {"must", "should"} + else "possible" if modal_word in {"can", "could", "may", "might", "would"} + else "asserted" + ) + return self._make( + subject=subject, + lemma="remain", + object_value=object_value, + complement=None, + qualifiers=extra_qualifiers, + modality=modality, + polarity=self._polarity(WORD.findall(main_surface)), + voice="active", + clause_type=clause_type, + evidence_row=evidence_row, + subject_span=subject_span, + predicate_span=predicate_span, + object_span=object_span, + domain_ids=domain_ids, + ) + + +class RepairedOpenSchemaConversationalTransducer(OpenSchemaConversationalTransducer): + @staticmethod + def _structural_operation(question: str, has_state: bool) -> str: + lower = question.casefold() + query = set(tokens(question)) + if ( + "count" in query + and query & {"record", "records", "event", "events", "observation", "observations", "entries"} + ): + return "count" + if re.search(r"\blook\s+up\b", lower): + return "lookup" + return OpenSchemaConversationalTransducer._structural_operation(question, has_state) + + +class RepairedNativeGeneralistAgent(NativePropositionKnowledgeAgent): + def __init__( + self, + specialist_checkpoint: Path, + knowledge_checkpoint: Path, + address_index_path: Path, + substrate_path: Path, + **kwargs: Any, + ) -> None: + kwargs.setdefault("agent_id", "oracle-stage5k33-repaired-native-generalist-agent") + super().__init__(specialist_checkpoint, knowledge_checkpoint, substrate_path, **kwargs) + self.address_index = CrossDomainAddressIndex.load(address_index_path) + self.knowledge_basis = CrossAddressedKnowledgeBasis(self.knowledge_basis, self.address_index) + self.proposition_compiler = K33PropositionCompiler() + + @classmethod + def _intent_lattice( + cls, + text: str, + context: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + audit = dict(super()._intent_lattice(text, context)) + scores = dict(audit["candidate_scores"]) + lower = clean(text).casefold() + words = set(tokens(text)) + role_predicates = ( + "carry", "carries", "contain", "contains", "hold", "holds", + "mean", "means", "represent", "represents", "store", "stores", + "transport", "transports", + ) + local_role_form = bool(re.search( + r"\bwhat\s+does\s+(?:a|an|the)?\s*(?:it|this|that|[a-z][a-z0-9_-]*)\s+" + r"(?:" + "|".join(role_predicates) + r")\b", + lower, + )) + scores["definition_role_query"] = 1.0 if local_role_form else 0.0 + data_domains = {"population", "populations", "earthquake", "earthquakes", "magnitude", "depth", "gdp"} + explicit_lookup = bool( + re.search(r"\blook\s+up\b", lower) + and words & data_domains + and words & {"data", "database", "dataset", "registered", "population", "earthquake", "earthquakes"} + ) + if explicit_lookup: + scores["bounded_dataset"] = 1.0 + scores["definition_query"] = 0.0 + audit["candidate_scores"] = scores + if audit.get("selected_intent_family") != "creative_generation_request": + audit["selected_intent_family"] = next( + (name for name in audit["precedence"] if scores.get(name, 0.0) > 0.0), + "open", + ) + audit["k33_intent_repairs"] = { + "definition_role_requires_relational_role_predicate": True, + "explicit_multiword_lookup_detected": explicit_lookup, + } + return audit + + def _handle_sensor( + self, + text: str, + context: Mapping[str, Any], + ) -> HandlerResult | None: + stream = context.get("sensor_stream") + if not isinstance(stream, Sequence) or isinstance(stream, (str, bytes)) or not stream: + return None + rows = [dict(value) for value in stream if isinstance(value, Mapping)] + if not rows: + return self._abstention("sensor_state", "the sensor state", "sensor_rows_invalid") + candidate_fields = [ + key for key, value in rows[0].items() + if key.casefold() not in {"time", "timestamp", "unit", "sensor", "sensor_id"} + and isinstance(value, (int, float)) + ] + if not candidate_fields: + return self._abstention("sensor_state", "the sensor state", "numeric_sensor_field_absent") + clauses: list[SemanticClause] = [] + directions: dict[str, str] = {} + stable_fields: list[str] = [] + locally_bounded_fields: list[str] = [] + deltas: dict[str, float] = {} + for field in candidate_fields: + values = [float(row[field]) for row in rows if isinstance(row.get(field), (int, float))] + if len(values) < 2: + continue + delta = values[-1] - values[0] + scale = max(1.0, *(abs(value) for value in values)) + direction_floor = max(1e-9, 1e-9 * scale) + bounded_tolerance = max(1e-6, 0.02 * scale) + direction = ( + "remain" if abs(delta) <= direction_floor + else "increase" if delta > 0 else "decrease" + ) + if max(values) - min(values) <= bounded_tolerance: + locally_bounded_fields.append(field) + unit = clean(rows[0].get("unit", "units")) + if field.endswith("_c") and unit == "units": + unit = "degrees C" + elif field.endswith("_kpa") and unit == "units": + unit = "kPa" + label = field.replace("_", " ") + directions[field] = direction + deltas[field] = delta + if direction == "remain": + stable_fields.append(field) + clauses.append(SemanticClause( + f"The {label}", "remain", complement="stable", + qualifiers=(("near", f"{values[-1]:g} {unit}"),), tense="past", + )) + else: + clauses.append(SemanticClause( + f"The {label}", direction, + qualifiers=(("from", f"{values[0]:g} {unit}"), ("to", f"{values[-1]:g} {unit}")), + tense="past", + )) + if not clauses: + return self._abstention("sensor_state", "the sensor trend", "insufficient_sensor_records") + primary = candidate_fields[0] + return HandlerResult( + "sensor_state", + "answered", + (("fields", candidate_fields), ("records", len(rows))), + ( + ("direction", directions.get(primary)), + ("delta", deltas.get(primary)), + ("directions", directions), + ("stable_fields", stable_fields), + ("locally_bounded_fields", locally_bounded_fields), + ), + tuple(clauses), + (("stream_digest", digest(rows)),), + ( + "bind_timestamped_readings", + "separate_direction_from_local_boundedness", + "compute_directional_state", + ), + tool_audit={ + "tool": "native_sensor_fold", + "read_only": True, + "records": len(rows), + "direction_floor_relative": 1e-9, + "local_boundedness_relative": 0.02, + }, + ) + + def _handle_tool(self, session_id: str, text: str) -> HandlerResult | None: + result = super()._handle_tool(session_id, text) + if result is None or result.status != "answered" or not result.tool_audit: + return result + operation = str(result.tool_audit.get("operation", "")) + values = tuple(str(value) for value in result.tool_audit.get("values", ())) + if operation == "count" and values: + return HandlerResult( + result.operation, + result.status, + result.arguments, + result.result, + (SemanticClause("The bounded expert", "contain", object_value=f"{values[0]} matching records"),), + result.evidence, + (*result.computation_trace, "realize_aggregate_count_without_row_placeholder"), + tool_audit=result.tool_audit, + memory_audit=result.memory_audit, + ) + if operation == "lookup" and values and "population" in set(tokens(text)): + country = next( + (value for value in ("Canada", "Mexico", "United States") if value.casefold() in text.casefold()), + "The selected country's", + ) + subject = f"{country}'s population" if country != "The selected country's" else country + " population" + return HandlerResult( + result.operation, + result.status, + result.arguments, + result.result, + (SemanticClause(subject, "be", complement=values[0]),), + result.evidence, + (*result.computation_trace, "realize_schema_field_as_named_population"), + tool_audit=result.tool_audit, + memory_audit=result.memory_audit, + ) + return result + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ablate_knowledge_cells: Sequence[str] = (), + ) -> dict[str, Any]: + output = super().answer( + session_id, + text, + context, + ablate_specialists=ablate_specialists, + ablate_knowledge_cells=ablate_knowledge_cells, + ) + output["schema_version"] = SCHEMA + output["cross_domain_address_index"] = { + "schema_version": self.address_index.metadata.get("schema_version"), + "index_sha256": self.address_index.file_sha256, + "primitive_domain_modes": self.address_index.metadata.get("primitive_domain_modes"), + "registered_two_mode_cells": self.address_index.metadata.get("registered_two_mode_cells"), + "adds_new_specialists": False, + "traditional_language_model_calls": 0, + "next_token_predictions": 0, + } + output["runtime"]["cross_domain_address_model_calls"] = 0 + output["runtime"]["cross_domain_address_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = [ + "CrossAddressedKnowledgeBasis", "K33PropositionCompiler", + "RepairedNativeGeneralistAgent", "RepairedOpenSchemaConversationalTransducer", +] diff --git a/root/wander-train/stage5k35_bidirectional_native_generalist_agent.py b/root/wander-train/stage5k35_bidirectional_native_generalist_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..406ec2983f44a781673ea395ee7ae02473c62aae --- /dev/null +++ b/root/wander-train/stage5k35_bidirectional_native_generalist_agent.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +"""K35 bidirectional specialist routing and broader native constructions. + +K34 showed that the minimal specialist basis was stable but its one-way router +could not always reach strongly matching evidence outside its first three +domain guesses, and that held-out development documents were never promoted +into the production retrieval index. K35 keeps the same ten primitive modes +and five cells. A high-coverage corpus address may feed back into the router, +activating the already-registered specialist that owns that address. It also +adds finite main-clause, leading-subordinate, anaphoric-state, and inflectional +definition constructions exposed by the prospective failures. +""" + +from __future__ import annotations + +import hashlib +import math +from pathlib import Path +import re +from typing import Any, Mapping, Sequence + +import numpy as np + +from stage5k15_native_mixed_agent import ( + HandlerResult, + NativeAnswerGrammar, + QuadraticAnswerInverse, + SemanticClause, + clean, + digest, + past, + sentence_case, + third_person, + tokens, +) +from stage5k29_general_knowledge_domains import PUBLIC_CORPUS, iter_records +from stage5k29_meta_fold_knowledge_basis import ( + STOP, + TrainedKnowledgeBasis, + _domain_score, + directional_text_state, +) +from stage5k31_native_proposition_compiler_agent import ( + AUXILIARIES, + FORM_TO_LEMMA, + MODALS, + NEGATIONS, + PREDICATE_PATTERN, + WORD, + _span_in, + _strip_surface, +) +from stage5k33_cross_domain_address_index import CrossDomainAddressIndex +from stage5k33_repaired_native_generalist_agent import ( + K33PropositionCompiler, + RepairedNativeGeneralistAgent, +) +from train_stage2a_role_ladder import HRR_DIM, cosine, normalize, unitary + + +SCHEMA = "oracle-stage5k35-bidirectional-native-generalist-agent-v1" + + +CANONICAL_TERMS = { + "below": "less", "under": "less", "stay": "remain", "stays": "remain", + "stayed": "remain", "yield": "produce", "yields": "produce", "yielded": "produce", + "earliest": "first", "countries": "country", "components": "component", + "substances": "substance", "entries": "record", "records": "record", + "atoms": "atom", "methods": "method", "energies": "energy", + "forms": "form", "formed": "form", "includes": "include", "included": "include", + "contains": "contain", "contained": "contain", "provides": "provide", + "produces": "produce", "produced": "produce", "travels": "travel", + "crosses": "cross", "enters": "enter", +} + + +def canonical_terms(value: str) -> set[str]: + return { + CANONICAL_TERMS.get(token, token) + for token in tokens(value) + if token not in STOP + } + + +class FullCorpusAddressedKnowledgeBasis: + """Promote all source-held rows after selection while preserving split audit.""" + + def __init__( + self, + basis: TrainedKnowledgeBasis, + address_index: CrossDomainAddressIndex, + ) -> None: + self.basis = basis + self.address_index = address_index + + def pool(self): + return self.basis.pool() + + def _candidates( + self, + query: str, + corpus_path: Path, + selected_domain_ids: Sequence[str] | None, + ) -> list[tuple[float, dict[str, Any]]]: + selected = None if selected_domain_ids is None else set(selected_domain_ids) + query_terms = canonical_terms(query) + candidates: list[tuple[float, dict[str, Any]]] = [] + for row in iter_records(corpus_path): + primary = str(row["required_domain_ids"][0]) + address_ids = self.address_index.addresses(str(row["text_sha256"]), primary) + matched = tuple(address_ids if selected is None else sorted(selected & set(address_ids))) + if not matched: + continue + row_terms = canonical_terms(str(row["text"])) + shared = query_terms & row_terms + if not shared: + continue + lexical = len(shared) / max(1.0, math.sqrt(len(query_terms) * len(row_terms))) + candidates.append((lexical, { + "domain_id": primary, + "addressed_domain_ids": list(address_ids), + "matched_selected_domain_ids": list(matched), + "text": row["text"], + "source_document": row["source_document"], + "source_document_sha256": row["source_document_sha256"], + "text_sha256": row["text_sha256"], + "license_note": row["license_note"], + "source_split": row["split"], + "retrieval_score": lexical, + "shared_terms": sorted(shared), + "query_term_coverage": len(shared) / max(1, len(query_terms)), + "evidence_term_coverage": len(shared) / max(1, len(row_terms)), + "answer_status": "evidence_only", + })) + return sorted(candidates, key=lambda item: (-item[0], item[1]["text_sha256"])) + + def best_global_candidate( + self, + query: str, + corpus_path: Path = PUBLIC_CORPUS, + ) -> dict[str, Any] | None: + values = self._candidates(query, corpus_path, None) + return values[0][1] if values else None + + def retrieve( + self, + query: str, + corpus_path: Path = PUBLIC_CORPUS, + top_k: int = 3, + *, + selected_domain_ids: Sequence[str] | None = None, + ) -> tuple[dict[str, Any], ...]: + selected = ( + tuple(selected_domain_ids) + if selected_domain_ids is not None + else tuple(self.pool().route(query)["selected_domain_ids"]) + ) + return tuple(value for _, value in self._candidates(query, corpus_path, selected)[:top_k]) + + +class BidirectionalKnowledgeSpecialistPool: + """Let a strong evidence address activate its existing domain specialist.""" + + def __init__(self, base_pool, knowledge_basis: FullCorpusAddressedKnowledgeBasis) -> None: + self.base_pool = base_pool + self.knowledge_basis = knowledge_basis + self.partition = base_pool.partition + self.profiles = base_pool.profiles + self.lexical_weight = base_pool.lexical_weight + + def route(self, text: str, *, ablate_cells: Sequence[str] = ()) -> dict[str, Any]: + routed = self.base_pool.route(text, ablate_cells=ablate_cells) + candidate = self.knowledge_basis.best_global_candidate(text) + eligible = bool( + candidate + and len(candidate.get("shared_terms", ())) >= 2 + and float(candidate.get("query_term_coverage", 0.0)) >= 0.50 + and float(candidate.get("retrieval_score", 0.0)) >= 0.18 + ) + selected_domains = list(routed["selected_domain_ids"]) + selected_cells = list(routed["selected_cell_ids"]) + correction_domain: str | None = None + correction_cell: str | None = None + blocked = set(ablate_cells) + if eligible and candidate is not None and not ( + set(candidate["addressed_domain_ids"]) & set(selected_domains) + ): + candidate_domains = [ + value for value in candidate["addressed_domain_ids"] + if value not in selected_domains + ] + query_state = directional_text_state(text) + scored = sorted( + ( + _domain_score(self.profiles[value], text, query_state, self.lexical_weight) + for value in candidate_domains if value in self.profiles + ), + key=lambda value: (-value.relevance, value.domain_id), + ) + for domain_score in scored: + cell = next(cell for cell in self.partition if domain_score.domain_id in cell) + cell_id = "__".join(cell) + if cell_id not in blocked: + correction_domain = domain_score.domain_id + correction_cell = cell_id + break + correction_applied = bool(correction_domain and correction_cell) + if correction_applied: + pairs = [(correction_cell, correction_domain)] + pairs.extend( + (cell, domain) + for cell, domain in zip(selected_cells, selected_domains) + if cell != correction_cell + ) + pairs = pairs[:3] + selected_cells = [value[0] for value in pairs] + selected_domains = [value[1] for value in pairs] + query_state = directional_text_state(text) + scores = tuple( + _domain_score(self.profiles[value], text, query_state, self.lexical_weight) + for value in selected_domains + ) + moving = normalize(sum( + (np.float32(max(value.relevance, 1e-4)) * value.state for value in scores), + np.zeros(HRR_DIM, dtype=np.complex64), + )) + root = unitary("stage5k29:protected-root:" + hashlib.sha256(clean(text).encode()).hexdigest()) + residual = moving - np.complex64(np.vdot(root, moving)) * root + protected = normalize(np.float32(4.0) * root + residual) + routed["fused_state"] = moving + routed["protected_root_cosine"] = cosine(root, protected) + routed["selected_cell_ids"] = selected_cells + routed["selected_domain_ids"] = selected_domains + routed["address_feedback"] = { + "candidate_available": candidate is not None, + "candidate_text_sha256": None if candidate is None else candidate["text_sha256"], + "candidate_addresses": [] if candidate is None else candidate["addressed_domain_ids"], + "candidate_query_term_coverage": 0.0 if candidate is None else candidate["query_term_coverage"], + "candidate_retrieval_score": 0.0 if candidate is None else candidate["retrieval_score"], + "eligible": eligible, + "correction_applied": correction_applied, + "activated_existing_domain": correction_domain, + "activated_existing_cell": correction_cell, + "new_specialist_created": False, + "answer_or_gold_labels_used": False, + } + return routed + + def ablation_changes(self, text: str) -> dict[str, Any]: + baseline = self.route(text) + ablated = self.route(text, ablate_cells=baseline["selected_cell_ids"]) + similarity = cosine(baseline["fused_state"], ablated["fused_state"]) + return { + "selected_set_changed": baseline["selected_cell_ids"] != ablated["selected_cell_ids"], + "fused_state_cosine": similarity, + "material_change": baseline["selected_cell_ids"] != ablated["selected_cell_ids"] and similarity < 0.95, + } + + +class K35PropositionCompiler(K33PropositionCompiler): + _DIRECT_RELATIONS = frozenset({ + "contain", "include", "provide", "produce", "form", "generate", "support", + "show", "maintain", "remain", "enable", "cause", "create", + }) + + def _direct_main_clause( + self, + surface: str, + evidence_row: Mapping[str, Any], + domain_ids: Sequence[str], + *, + clause_type: str, + conditional: bool, + extra_qualifiers: Sequence[tuple[str, str]], + base_offset: int, + ): + relative = re.search(r"\b(?:that|which|who)\b", surface, re.IGNORECASE) + boundary = relative.start() if relative is not None else len(surface) + matches = [ + value for value in PREDICATE_PATTERN.finditer(surface) + if value.start() < boundary + and FORM_TO_LEMMA[value.group(0).casefold()] in self._DIRECT_RELATIONS + and _strip_surface(surface[:value.start()]) + ] + if not matches: + return None + match = matches[0] + lemma = FORM_TO_LEMMA[match.group(0).casefold()] + left = surface[:match.start()] + right = self._clean_argument(surface[match.end():]) + left = re.sub( + r"\s+(?:readily|usually|generally|normally|always|therefore|then)\s*$", + "", + left, + flags=re.IGNORECASE, + ) + left = re.sub(r",\s*[A-Za-z0-9_.-]+,\s*$", "", left) + modal_match = re.search( + r"\b(?:can|could|may|might|must|shall|should|will|would)\b", + left, + re.IGNORECASE, + ) + subject_surface = left[:modal_match.start()] if modal_match is not None else left + subject = self._clean_argument(subject_surface) + if not subject: + return None + qualifiers = list(extra_qualifiers) + object_value: str | None = right or None + because = re.match(r"^because\s+(.+)$", right, re.IGNORECASE) + if because is not None: + object_value = None + qualifiers.append(("because", self._clean_argument(because.group(1)))) + evidence = clean(evidence_row.get("fact", evidence_row.get("text", ""))) + subject_span = _span_in(evidence, subject, base_offset) + predicate_span = _span_in(evidence, match.group(0), base_offset) + object_span = _span_in( + evidence, + object_value, + predicate_span[1] if predicate_span is not None else base_offset, + ) + if subject_span is None or predicate_span is None: + return None + left_words = WORD.findall(left) + return self._make( + subject=subject, + lemma=lemma, + object_value=object_value, + complement=None, + qualifiers=qualifiers, + modality=self._modality(left_words, conditional), + polarity=self._polarity(left_words), + voice="active", + clause_type=clause_type, + evidence_row=evidence_row, + subject_span=subject_span, + predicate_span=predicate_span, + object_span=object_span, + domain_ids=domain_ids, + ) + + def _parse_simple( + self, + clause: str, + evidence_row: Mapping[str, Any], + domain_ids: Sequence[str], + *, + clause_type: str = "assertion", + conditional: bool = False, + inherited_subject: str | None = None, + extra_qualifiers: Sequence[tuple[str, str]] = (), + base_offset: int = 0, + ): + surface = _strip_surface(clause) + leading = re.match( + r"^(Once|When|After|Before)\s+(.+?),\s*(.+)$", + surface, + re.IGNORECASE, + ) + if leading is not None and inherited_subject is None: + qualifier = "when" if leading.group(1).casefold() in {"once", "when"} else leading.group(1).casefold() + return self._parse_simple( + leading.group(3), + evidence_row, + domain_ids, + clause_type=clause_type, + conditional=conditional, + extra_qualifiers=(*extra_qualifiers, (qualifier, self._clean_argument(leading.group(2)))), + base_offset=base_offset + leading.start(3), + ) + if inherited_subject is None: + direct = self._direct_main_clause( + surface, + evidence_row, + domain_ids, + clause_type=clause_type, + conditional=conditional, + extra_qualifiers=extra_qualifiers, + base_offset=base_offset, + ) + if direct is not None: + return direct + return super()._parse_simple( + clause, + evidence_row, + domain_ids, + clause_type=clause_type, + conditional=conditional, + inherited_subject=inherited_subject, + extra_qualifiers=extra_qualifiers, + base_offset=base_offset, + ) + + +class K35AnswerGrammar(NativeAnswerGrammar): + """Inflect transparent plural noun phrases without a statistical model.""" + + @staticmethod + def _plural(subject: str) -> bool: + lower = clean(subject).casefold() + if lower in {"they", "we", "you"} or " and " in lower: + return True + final = re.sub(r"[^a-z]", "", lower.split()[-1]) if lower.split() else "" + return bool( + len(final) > 2 + and final.endswith("s") + and not final.endswith(("ss", "us", "is")) + and final not in {"news", "physics", "mathematics", "economics"} + ) + + @classmethod + def clause(cls, clause: SemanticClause) -> str: + subject = sentence_case(clause.subject) + lemma = clean(clause.predicate).casefold() + is_first_person = subject.casefold() == "i" + plural = cls._plural(subject) + if clause.modal: + verb = f"{clause.modal} {lemma}" + elif clause.tense == "past": + verb = "were" if lemma == "be" and plural and not is_first_person else past(lemma) + elif is_first_person: + verb = "am" if lemma == "be" else lemma + elif plural: + verb = "are" if lemma == "be" else lemma + else: + verb = third_person(lemma) + if clause.polarity == "negative" and clause.modal is None: + auxiliary = "did not" if clause.tense == "past" else ("do not" if plural else "does not") + verb = f"{auxiliary} {lemma}" + parts = [subject, verb] + if clause.object_value: + parts.append(clean(clause.object_value)) + if clause.complement: + parts.append(clean(clause.complement)) + for preposition, value in clause.qualifiers: + parts.extend((clean(preposition), clean(value))) + return " ".join(parts).rstrip(".?!") + "." + + +class BidirectionalNativeGeneralistAgent(RepairedNativeGeneralistAgent): + def __init__( + self, + specialist_checkpoint: Path, + knowledge_checkpoint: Path, + address_index_path: Path, + substrate_path: Path, + **kwargs: Any, + ) -> None: + kwargs.setdefault("agent_id", "oracle-stage5k35-bidirectional-native-generalist-agent") + super().__init__( + specialist_checkpoint, + knowledge_checkpoint, + address_index_path, + substrate_path, + **kwargs, + ) + full_basis = FullCorpusAddressedKnowledgeBasis( + self.knowledge_basis.basis, + self.address_index, + ) + self.knowledge_basis = full_basis + self.knowledge_specialists = BidirectionalKnowledgeSpecialistPool( + self.knowledge_specialists, + full_basis, + ) + self.proposition_compiler = K35PropositionCompiler() + self.inverse = QuadraticAnswerInverse(K35AnswerGrammar()) + + @staticmethod + def _definition_action(meaning: str): + value = RepairedNativeGeneralistAgent._definition_action(meaning) + if value is None: + return None + verb, theme, preposition, oblique = value + if verb.endswith("s") and not verb.endswith("ss"): + verb = verb[:-1] + return verb, theme, preposition, oblique + + @classmethod + def _intent_lattice( + cls, + text: str, + context: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + audit = dict(super()._intent_lattice(text, context)) + scores = dict(audit["candidate_scores"]) + lower = clean(text).casefold() + words = set(tokens(text)) + anaphoric_state = bool( + re.search(r"\bwhich\b.+\bchanged\b.+\bwhich\b.+\bdid\s+not\b", lower) + or re.search(r"\bwhich\s+(?:one|quantity).+\bdid\s+not\b", lower) + ) + imperative_count = bool( + "count" in words + and words & {"record", "records", "entry", "entries", "event", "events", "observation", "observations"} + and words & {"earthquake", "earthquakes", "population", "magnitude", "depth", "gdp"} + ) + if anaphoric_state: + scores["state_query"] = 1.0 + scores["definition_query"] = 0.0 + if imperative_count: + scores["bounded_dataset"] = 1.0 + scores["definition_query"] = 0.0 + audit["candidate_scores"] = scores + if audit.get("selected_intent_family") != "creative_generation_request": + audit["selected_intent_family"] = next( + (name for name in audit["precedence"] if scores.get(name, 0.0) > 0.0), + "open", + ) + audit["k35_intent_repairs"] = { + "anaphoric_state_query": anaphoric_state, + "imperative_count": imperative_count, + } + return audit + + def _handle_event(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + lower = clean(text).casefold() + if re.search(r"\bwhich\b.+\bchanged\b.+\bdid\s+not\b", lower): + payloads, memory_audit = self._retrieve( + session_id, + {"situation": "state_contrast"}, + turn_id, + top_k=16, + ) + match = next((value for value in payloads if value.get("situation") == "state_contrast"), None) + if match: + return HandlerResult( + "state_contrast", + "answered", + (), + ( + ("changed", match["source"]), + ("transition", match["transition"]), + ("invariant", match["field"]), + ("invariant_state", match["result"]), + ), + ( + SemanticClause(match["source"], match["transition"], tense="past"), + SemanticClause(match["field"], "remain", complement=match["result"], tense="past"), + ), + (("memory_snapshot", memory_audit["snapshot_digest"]),), + ( + "resolve_anaphoric_state_contrast", + "retrieve_state_contrast", + "release_transition_and_invariant", + ), + memory_audit=memory_audit, + ) + return super()._handle_event(session_id, text, turn_id) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ablate_knowledge_cells: Sequence[str] = (), + ) -> dict[str, Any]: + output = super().answer( + session_id, + text, + context, + ablate_specialists=ablate_specialists, + ablate_knowledge_cells=ablate_knowledge_cells, + ) + output["schema_version"] = SCHEMA + output["runtime"]["bidirectional_address_feedback_model_calls"] = 0 + output["runtime"]["bidirectional_address_feedback_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = [ + "BidirectionalKnowledgeSpecialistPool", "BidirectionalNativeGeneralistAgent", + "FullCorpusAddressedKnowledgeBasis", "K35AnswerGrammar", "K35PropositionCompiler", "canonical_terms", +] diff --git a/root/wander-train/stage5k39_native_concept_address_index.py b/root/wander-train/stage5k39_native_concept_address_index.py new file mode 100644 index 0000000000000000000000000000000000000000..3428345de2ce92d055451aeb11f4d7a6fa2ce646 --- /dev/null +++ b/root/wander-train/stage5k39_native_concept_address_index.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""Compile/load a native WordNet-to-corpus relational concept address index. + +The checkpoint contains no learned embedding vectors. WordNet senses, +hypernym paths, derivational families, and definition terms become sparse +relational atoms. Runtime HRR states are generated deterministically from +those atoms; sparse postings retain exact interpretability and avoid scanning +the public corpus. +""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +from dataclasses import dataclass +import gzip +import hashlib +import io +import json +from pathlib import Path +import time +from typing import Any, Iterable, Mapping + +from stage5j1_native_contextual_sense_lattice import CONTENT_STOP, stem, tokens +from stage5k37_inverted_relational_index import InvertedRelationalIndex + + +SCHEMA = "oracle-stage5k39-native-concept-address-index-v1" +MAX_SENSES_PER_LEMMA = 8 +MAX_GLOSS_TERMS_PER_SENSE = 12 +MAX_CORPUS_TERMS_PER_ATOM = 512 +ROW_POSTING_WEIGHT_MIN = 0.55 + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def normalize_lexeme(value: str) -> str: + return " ".join(stem(part) for part in str(value).replace("_", " ").casefold().split()) + + +def content_terms(value: str) -> tuple[str, ...]: + return tuple(dict.fromkeys( + stem(token) for token in tokens(value) + if stem(token) not in CONTENT_STOP and len(stem(token)) > 2 + )) + + +def set_max(profile: dict[str, float], atom: str, weight: float) -> None: + profile[atom] = max(float(weight), profile.get(atom, 0.0)) + + +class UnionFind: + def __init__(self) -> None: + self.parent: dict[str, str] = {} + + def find(self, value: str) -> str: + self.parent.setdefault(value, value) + if self.parent[value] != value: + self.parent[value] = self.find(self.parent[value]) + return self.parent[value] + + def union(self, left: str, right: str) -> None: + a, b = self.find(left), self.find(right) + if a != b: + low, high = sorted((a, b)) + self.parent[high] = low + + def canonical_groups(self) -> dict[str, str]: + members: dict[str, list[str]] = defaultdict(list) + for value in self.parent: + members[self.find(value)].append(value) + result: dict[str, str] = {} + for values in members.values(): + label = min(values) + for value in values: + result[value] = label + return result + + +@dataclass(frozen=True) +class NativeConceptAddressIndex: + lexeme_profiles: Mapping[str, tuple[tuple[str, float], ...]] + corpus_terms_by_atom: Mapping[str, tuple[str, ...]] + metadata: Mapping[str, Any] + file_sha256: str + + @classmethod + def load(cls, path: Path) -> "NativeConceptAddressIndex": + with gzip.open(path, "rt", encoding="utf-8") as handle: + value = json.load(handle) + if value.get("schema_version") != SCHEMA: + raise ValueError("unsupported K39 native concept address index") + profiles = value.get("lexeme_profiles") + postings = value.get("corpus_terms_by_atom") + if not isinstance(profiles, Mapping) or not isinstance(postings, Mapping): + raise ValueError("K39 concept checkpoint is missing profiles or postings") + return cls( + lexeme_profiles={ + str(lexeme): tuple((str(atom), float(weight)) for atom, weight in atoms) + for lexeme, atoms in profiles.items() + }, + corpus_terms_by_atom={ + str(atom): tuple(str(term) for term in terms) + for atom, terms in postings.items() + }, + metadata={ + key: item for key, item in value.items() + if key not in {"lexeme_profiles", "corpus_terms_by_atom"} + }, + file_sha256=sha256(path), + ) + + def profile(self, surface: str) -> tuple[tuple[str, float], ...]: + value = normalize_lexeme(surface) + candidates = tuple(dict.fromkeys((value, stem(value), value.replace("-", " ")))) + for candidate in candidates: + profile = self.lexeme_profiles.get(candidate) + if profile: + return profile + fallback = tuple(dict.fromkeys(( + ("l:" + value, 1.0), + ("m:" + stem(value), 0.90), + ))) + return fallback + + +def compile_index( + wordnet_data: Path, + wordnet_source: Path, + relational_index_path: Path, + output_path: Path, +) -> dict[str, Any]: + started = time.perf_counter() + import nltk + from nltk.corpus import wordnet as wn + + nltk.data.path.insert(0, str(wordnet_data.resolve())) + relational = InvertedRelationalIndex.load(relational_index_path) + corpus_terms = tuple(sorted(relational.term_postings)) + + profiles: dict[str, dict[str, float]] = defaultdict(dict) + sense_counts: dict[str, int] = defaultdict(int) + derivations = UnionFind() + synsets = 0 + lemma_memberships = 0 + for synset in wn.all_synsets(): + synsets += 1 + own_atom = "c:" + synset.name() + hyper1 = tuple(dict.fromkeys((*synset.hypernyms(), *synset.instance_hypernyms()))) + hyper2 = tuple(dict.fromkeys( + parent for value in hyper1 + for parent in (*value.hypernyms(), *value.instance_hypernyms()) + )) + gloss = content_terms(synset.definition())[:MAX_GLOSS_TERMS_PER_SENSE] + lemmas = synset.lemmas() + normalized_lemmas = tuple(dict.fromkeys(normalize_lexeme(value.name()) for value in lemmas)) + for lemma in lemmas: + lexeme = normalize_lexeme(lemma.name()) + if sense_counts[lexeme] >= MAX_SENSES_PER_LEMMA: + continue + sense_counts[lexeme] += 1 + lemma_memberships += 1 + profile = profiles[lexeme] + set_max(profile, "l:" + lexeme, 1.0) + set_max(profile, "m:" + stem(lexeme), 0.90) + set_max(profile, own_atom, 1.0) + set_max(profile, "x:" + synset.lexname(), 0.12) + for parent in hyper1: + set_max(profile, "c:" + parent.name(), 0.78) + for parent in hyper2: + set_max(profile, "c:" + parent.name(), 0.58) + for neighbor in normalized_lemmas: + set_max(profile, "l:" + neighbor, 0.92) + for term in gloss: + set_max(profile, "l:" + term, 0.24) + for related in lemma.derivationally_related_forms(): + other = normalize_lexeme(related.name()) + if other: + derivations.union(lexeme, other) + + for lexeme, group in derivations.canonical_groups().items(): + if lexeme in profiles: + set_max(profiles[lexeme], "d:" + group, 0.82) + + corpus_terms_by_atom: dict[str, list[str]] = defaultdict(list) + corpus_terms_with_dictionary_profile = 0 + for term in corpus_terms: + normalized = normalize_lexeme(term) + profile = profiles.get(normalized) + if profile: + corpus_terms_with_dictionary_profile += 1 + atoms = profile.items() + else: + atoms = (("l:" + normalized, 1.0), ("m:" + stem(normalized), 0.90)) + for atom, weight in atoms: + if weight >= ROW_POSTING_WEIGHT_MIN and not atom.startswith("x:"): + corpus_terms_by_atom[atom].append(term) + + pruned_atoms = { + atom: len(values) + for atom, values in corpus_terms_by_atom.items() + if len(set(values)) > MAX_CORPUS_TERMS_PER_ATOM + } + postings = { + atom: sorted(set(values)) + for atom, values in sorted(corpus_terms_by_atom.items()) + if atom not in pruned_atoms + } + serialized_profiles = { + lexeme: [[atom, round(weight, 4)] for atom, weight in sorted(profile.items())] + for lexeme, profile in sorted(profiles.items()) + } + value = { + "schema_version": SCHEMA, + "status": "compiled_native_dictionary_concept_addresses", + "construction": ( + "WordNet sense, synonym, two-level hypernym, derivational-family, definition-term, " + "morphology, and K37 corpus-term postings" + ), + "wordnet_source_sha256": sha256(wordnet_source), + "relational_index_sha256": sha256(relational_index_path), + "public_corpus_sha256": relational.metadata["public_corpus_sha256"], + "synsets": synsets, + "lexeme_profiles": serialized_profiles, + "corpus_terms": len(corpus_terms), + "corpus_terms_with_dictionary_profile": corpus_terms_with_dictionary_profile, + "lemma_sense_memberships": lemma_memberships, + "maximum_senses_per_lemma": MAX_SENSES_PER_LEMMA, + "maximum_gloss_terms_per_sense": MAX_GLOSS_TERMS_PER_SENSE, + "row_posting_weight_min": ROW_POSTING_WEIGHT_MIN, + "maximum_corpus_terms_per_atom": MAX_CORPUS_TERMS_PER_ATOM, + "pruned_high_frequency_atoms": dict(sorted(pruned_atoms.items())), + "corpus_terms_by_atom": postings, + "concept_atoms_with_postings": len(postings), + "concept_posting_memberships": sum(len(values) for values in postings.values()), + "primitive_domain_modes": int(relational.metadata["primitive_domain_modes"]), + "registered_knowledge_cells": int(relational.metadata["registered_knowledge_cells"]), + "adds_new_specialists": False, + "stabilizing_spine_gain": 4.0, + "traditional_language_model_calls": 0, + "transformer_calls": 0, + "pretrained_embedding_encoder_calls": 0, + "next_token_predictions": 0, + "global_gradient_steps": 0, + } + if output_path.exists(): + raise FileExistsError(f"K39 concept checkpoint already exists: {output_path}") + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("xb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + with io.TextIOWrapper(compressed, encoding="utf-8") as handle: + json.dump(value, handle, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + handle.write("\n") + return { + **{key: item for key, item in value.items() if key not in { + "lexeme_profiles", "corpus_terms_by_atom", "pruned_high_frequency_atoms", + }}, + "lexeme_profile_count": len(serialized_profiles), + "pruned_high_frequency_atom_count": len(pruned_atoms), + "output_path": str(output_path), + "output_bytes": output_path.stat().st_size, + "output_sha256": sha256(output_path), + "elapsed_seconds": time.perf_counter() - started, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--wordnet-data", type=Path, required=True) + parser.add_argument("--wordnet-source", type=Path, required=True) + parser.add_argument("--relational-index", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(compile_index( + args.wordnet_data, args.wordnet_source, args.relational_index, args.output, + ), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() + + +__all__ = ["NativeConceptAddressIndex", "compile_index", "normalize_lexeme"] diff --git a/root/wander-train/stage5k3_language_adapter.py b/root/wander-train/stage5k3_language_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..b0ffac259a2dad7e857a388043f51b03b5f71964 --- /dev/null +++ b/root/wander-train/stage5k3_language_adapter.py @@ -0,0 +1,987 @@ +#!/usr/bin/env python3 +"""Stage 5K3 language-neutral adapter contract and English adapter A. + +Human languages are coordinate charts over the shared relational geometry; +none is the canonical semantic language. Adapters expose ambiguity-retaining +program hypotheses, surface-to-role bindings, grammatical residuals, and +confidence. Canonical operator identity is structural and contains no surface +lexeme. + +The English adapter is intentionally bounded. It connects the existing native +construction algebra and the Stage 5K2 VerbNet operator basis to Stage 5K1. It +does not claim unrestricted parsing. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from abc import ABC, abstractmethod +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from typing import Any, Mapping, Sequence + +from stage5j1_native_contextual_sense_lattice import WORD, stem +from stage5j2_native_english_construction_algebra import ( + COMPLEMENTIZERS, + COORDINATORS, + DETERMINERS, + LexicalCategoryResolver, + PREPOSITIONS, + PredicateArgumentShape, + predicate_argument_shape, +) +from stage5k1_generative_relational_workspace import ( + EphemeralRelationalWorkspace, + Provenance, + RelationalConstraint, + RelationalState, + RelationalValue, +) +from stage5k2_verbnet_operator_induction import ( + FrameProgram, + OperatorInvocation, + VerbNetOperatorBasis, + clean, +) + + +SCHEMA = "oracle-stage5k3-language-adapter-contract-v1" +ANALYSIS_SCHEMA = "oracle-stage5k3-language-analysis-v1" +PROGRAM_SCHEMA = "oracle-stage5k3-instantiated-program-v1" + + +def canonical(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class SurfaceBinding: + language_tag: str + role: str + surface: str = field(repr=False, compare=False) + surface_sha256: str + value_type: str + concept_id: str | None + token_start: int + token_stop: int + confidence: float + + @classmethod + def create( + cls, + language_tag: str, + role: str, + surface: str, + *, + value_type: str = "entity", + concept_id: str | None = None, + token_start: int = -1, + token_stop: int = -1, + confidence: float = 1.0, + ) -> "SurfaceBinding": + normalized = " ".join(str(surface).strip().split()) + return cls( + str(language_tag), + str(role), + normalized, + hashlib.sha256(normalized.casefold().encode("utf-8")).hexdigest(), + str(value_type), + None if concept_id is None else str(concept_id), + int(token_start), + int(token_stop), + float(confidence), + ) + + def as_dict(self) -> dict[str, Any]: + return { + "language_tag": self.language_tag, + "role": self.role, + "surface_sha256": self.surface_sha256, + "value_type": self.value_type, + "concept_id_sha256": ( + None + if self.concept_id is None + else hashlib.sha256(self.concept_id.encode("utf-8")).hexdigest() + ), + "token_start": self.token_start, + "token_stop": self.token_stop, + "confidence": self.confidence, + "stored_plaintext_surface": False, + } + + +@dataclass(frozen=True) +class AdapterResidual: + feature: str + value: str + obligatory: bool + confidence: float + + def as_dict(self) -> dict[str, Any]: + return { + "feature": self.feature, + "value": self.value, + "obligatory": self.obligatory, + "confidence": self.confidence, + } + + +@dataclass(frozen=True) +class RelationalProgramHypothesis: + hypothesis_id: str + language_tag: str + frame_id: str + transformation_family_id: str + predicate_token_index: int + predicate_surface_sha256: str + lemma: str + canonical_operator_ids: tuple[str, ...] + invocations: tuple[OperatorInvocation, ...] = field(repr=False, compare=False) + surface_bindings: tuple[SurfaceBinding, ...] = field(repr=False, compare=False) + unresolved_roles: tuple[str, ...] + residuals: tuple[AdapterResidual, ...] + score: float + construction_confidence: float + + def binding_map(self) -> dict[str, SurfaceBinding]: + return {value.role: value for value in self.surface_bindings} + + @property + def language_invariant_program_sha256(self) -> str: + return digest({ + "operators": self.canonical_operator_ids, + "bindings": sorted( + ( + value.role, + value.concept_id + if value.concept_id is not None + else "unmapped:" + value.surface_sha256, + ) + for value in self.surface_bindings + ), + "unresolved_roles": self.unresolved_roles, + }) + + def as_dict(self) -> dict[str, Any]: + return { + "hypothesis_id": self.hypothesis_id, + "language_tag": self.language_tag, + "frame_id": self.frame_id, + "transformation_family_id": self.transformation_family_id, + "predicate_token_index": self.predicate_token_index, + "predicate_surface_sha256": self.predicate_surface_sha256, + "lemma_sha256": hashlib.sha256(self.lemma.encode("utf-8")).hexdigest(), + "canonical_operator_ids": list(self.canonical_operator_ids), + "surface_bindings": [value.as_dict() for value in self.surface_bindings], + "unresolved_roles": list(self.unresolved_roles), + "residuals": [value.as_dict() for value in self.residuals], + "score": self.score, + "construction_confidence": self.construction_confidence, + "language_invariant_program_sha256": ( + self.language_invariant_program_sha256 + ), + "english_pivot_used": False, + "surface_lexeme_defines_operator_identity": False, + } + + +@dataclass(frozen=True) +class LanguageAnalysis: + language_tag: str + surface_sha256: str + token_count: int + hypotheses: tuple[RelationalProgramHypothesis, ...] + adapter_id: str + adapter_semantic_sha256: str + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": ANALYSIS_SCHEMA, + "language_tag": self.language_tag, + "surface_sha256": self.surface_sha256, + "token_count": self.token_count, + "hypotheses": [value.as_dict() for value in self.hypotheses], + "adapter_id": self.adapter_id, + "adapter_semantic_sha256": self.adapter_semantic_sha256, + "canonical_language": None, + "english_pivot_used": False, + "ambiguity_retained": len(self.hypotheses) > 1, + } + + +@dataclass(frozen=True) +class InstantiatedAdapterProgram: + hypothesis: RelationalProgramHypothesis + atom_state_ids: tuple[str, ...] + result_state: RelationalState + generated_variable_ids: tuple[str, ...] + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": PROGRAM_SCHEMA, + "hypothesis": self.hypothesis.as_dict(), + "atom_state_ids": list(self.atom_state_ids), + "result_state": self.result_state.as_dict(), + "generated_variable_ids": list(self.generated_variable_ids), + "recursive_composition_used": len(self.atom_state_ids) > 1, + "ontology_insertion_count": 0, + } + + +class LanguageAdapter(ABC): + language_tags: tuple[str, ...] + adapter_id: str + + @abstractmethod + def analyze( + self, + text: str, + *, + maximum_hypotheses: int = 8, + ) -> LanguageAnalysis: + raise NotImplementedError + + @abstractmethod + def instantiate( + self, + hypothesis: RelationalProgramHypothesis, + workspace: EphemeralRelationalWorkspace, + *, + world_id: str | None = None, + ttl_turns: int | None = None, + ) -> InstantiatedAdapterProgram: + raise NotImplementedError + + +class LanguageAdapterRegistry: + def __init__(self) -> None: + self._adapters: dict[str, LanguageAdapter] = {} + + def register(self, adapter: LanguageAdapter) -> None: + for tag in adapter.language_tags: + normalized = str(tag).casefold() + existing = self._adapters.get(normalized) + if existing is not None and existing is not adapter: + raise ValueError(f"language adapter already registered for {tag}") + self._adapters[normalized] = adapter + + def adapter(self, language_tag: str) -> LanguageAdapter: + try: + return self._adapters[str(language_tag).casefold()] + except KeyError as exc: + raise KeyError(f"no adapter registered for {language_tag}") from exc + + @property + def language_tags(self) -> tuple[str, ...]: + return tuple(sorted(self._adapters)) + + def as_dict(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA, + "language_tags": list(self.language_tags), + "adapters": sorted({value.adapter_id for value in self._adapters.values()}), + "canonical_language": None, + "english_pivot_used": False, + } + + +class EnglishRelationalAdapter(LanguageAdapter): + """Native English coordinate chart over the shared K1/K2 geometry.""" + + language_tags = ("en", "en-us", "en-gb") + adapter_id = "oracle-stage5k3-english-adapter-a-v3-clause-lift" + + def __init__( + self, + wordnet: Any, + basis: VerbNetOperatorBasis, + *, + concept_lexicon: Mapping[str, str] | None = None, + sense_scorer: Any | None = None, + verbnet_structure: Any | None = None, + ): + self.wordnet = wordnet + self.basis = basis + self.resolver = LexicalCategoryResolver(wordnet) + self._frames = {frame.frame_id: frame for frame in basis.frames} + self._lemma_frames: dict[str, list[FrameProgram]] = defaultdict(list) + for frame in basis.frames: + for lemma in frame.lemmas: + self._lemma_frames[clean(lemma)].append(frame) + for lemma in self._lemma_frames: + self._lemma_frames[lemma].sort(key=lambda value: value.frame_id) + self._support = basis.template_support() + self.concept_lexicon = { + clean(key): str(value) + for key, value in (concept_lexicon or {}).items() + } + if (sense_scorer is None) != (verbnet_structure is None): + raise ValueError( + "sense scorer and VerbNet structure must be supplied together" + ) + self.sense_scorer = sense_scorer + self.verbnet_structure = verbnet_structure + self.semantic_sha256 = digest({ + "adapter_id": self.adapter_id, + "basis_source": basis.source_sha256, + "template_ids": sorted(basis.templates), + "rules": ( + "stage5j2-native-english-construction-algebra-v3-clause-lift+" + "stage5k3-geometric-observation-constraints-v3-clause-fillers" + ), + "sense_scorer_sha256": getattr(sense_scorer, "file_sha256", None), + "verbnet_structure_sha256": getattr( + verbnet_structure, "file_sha256", None, + ), + }) + + def install_operators( + self, + workspace: EphemeralRelationalWorkspace, + *, + minimum_support: int = 1, + ) -> tuple[str, ...]: + return self.basis.register_supported_operators( + workspace.canonical, minimum_support=minimum_support, + ) + + def _lemma(self, surface: str) -> str: + normalized = surface.casefold().replace("-", "_") + result = self.wordnet.morphy(normalized, self.wordnet.VERB) + return clean(result or stem(normalized)) + + def _predicate_indexes(self, values: Sequence[str]) -> tuple[int, ...]: + """Reject noun-position homographs without forcing one semantic frame.""" + + result: list[int] = [] + for index, surface in enumerate(values): + if not self.resolver.verb_like(surface): + continue + if index and values[index - 1] in DETERMINERS | PREPOSITIONS: + continue + if result: + boundary = values[result[-1] + 1:index] + if not any( + value in COORDINATORS or value in COMPLEMENTIZERS + for value in boundary + ): + continue + result.append(index) + return tuple(result) + + @staticmethod + def _observed_slots(shape: PredicateArgumentShape) -> tuple[str, ...]: + result: list[str] = [] + if shape.subject: + result.append("subject") + if shape.direct_object or ( + shape.clause_complement and shape.clause_preposition is None + ): + result.append("direct_object") + result.extend("prep:" + prep for prep, _ in shape.preposition_objects) + return tuple(result) + + @staticmethod + def _prep_matches(specification: str, actual: str) -> bool: + expected = specification.removeprefix("prep:") + if expected.startswith("restriction:") or expected == "unspecified": + return True + alternatives = { + value for value in re_split(expected) if value + } + return actual in alternatives + + def _frame_score( + self, + frame: FrameProgram, + shape: PredicateArgumentShape, + class_scores: Mapping[str, float] | None = None, + ) -> float: + observed = self._observed_slots(shape) + observed_counter = Counter( + value.split(":", 1)[0] if value.startswith("prep:") else value + for value in observed + ) + matched = 0.0 + possible = 0.0 + for slot, _ in frame.surface_slots: + possible += 1.0 + if slot == "subject" and shape.subject: + matched += 1.0 + elif slot == "direct_object" and shape.direct_object: + matched += 1.0 + elif slot.startswith("prep:") and any( + self._prep_matches(slot, prep) + for prep, _ in shape.preposition_objects + ): + matched += 1.0 + elif slot.startswith("object:") and shape.direct_object: + matched += 0.25 + coverage = matched / max(1.0, possible) + excess = max(0, len(observed) - int(possible)) + excess_penalty = min(0.25, 0.08 * excess) + mean_support = ( + sum(math.log1p(self._support[value.template_id]) for value in frame.invocations) + / max(1, len(frame.invocations)) + ) + support_score = min(1.0, mean_support / math.log1p(700.0)) + semantic_presence = 1.0 if frame.invocations else 0.0 + if class_scores is None: + score = ( + 0.62 * coverage + + 0.18 * shape.confidence + + 0.12 * support_score + + 0.08 * semantic_presence + ) + else: + score = ( + 0.48 * coverage + + 0.15 * shape.confidence + + 0.08 * support_score + + 0.04 * semantic_presence + + 0.25 * float(class_scores.get(frame.class_id, 0.0)) + ) + return max(0.0, min(1.0, score - excess_penalty)) + + def _sense_class_scores( + self, + text: str, + predicate_surface: str, + token_index: int, + lemma: str, + ) -> dict[str, float] | None: + if self.sense_scorer is None or self.verbnet_structure is None: + return None + candidates = self.sense_scorer.candidate_components( + text, predicate_surface, token_index, self.wordnet, + ) + if not candidates: + return {} + maximum = max(float(value.get("score", 0.0)) for value in candidates) + weights = [ + math.exp(max(-40.0, min( + 0.0, float(value.get("score", 0.0)) - maximum, + ))) + for value in candidates + ] + denominator = max(sum(weights), 1e-12) + result: dict[str, float] = {} + for item, weight in zip(candidates, weights): + try: + synset = self.wordnet.synset(item["sense"]) + except Exception: + continue + class_ids, _, mapping_confidence = ( + self.verbnet_structure.mapped_class_ids(synset, lemma) + ) + probability = weight / denominator + for class_id in class_ids: + result[class_id] = max( + result.get(class_id, 0.0), + probability * mapping_confidence, + ) + return result + + @staticmethod + def _slot_surface( + slot: str, + shape: PredicateArgumentShape, + ) -> str | None: + if slot == "subject": + return shape.subject + if slot == "direct_object": + return ( + shape.clause_complement + if shape.clause_complement is not None + and shape.clause_preposition is None + else shape.direct_object + ) + if slot.startswith("object:"): + return shape.indirect_object + if slot.startswith("prep:"): + if ( + shape.clause_complement is not None + and shape.clause_preposition is not None + and EnglishRelationalAdapter._prep_matches( + slot, shape.clause_preposition, + ) + ): + return shape.clause_complement + for prep, surface in shape.preposition_objects: + if EnglishRelationalAdapter._prep_matches(slot, prep): + return surface + return None + + def _bindings( + self, + frame: FrameProgram, + shape: PredicateArgumentShape, + ) -> tuple[tuple[SurfaceBinding, ...], tuple[str, ...]]: + bindings: dict[str, SurfaceBinding] = {} + for slot, role in frame.surface_slots: + surface = self._slot_surface(slot, shape) + if surface is None or role in bindings: + continue + bindings[role] = SurfaceBinding.create( + "en", role, surface, + value_type=( + "clause" + if shape.clause_complement is not None + and surface == shape.clause_complement + else "entity" + ), + concept_id=self.concept_lexicon.get(clean(surface)), + confidence=shape.confidence, + ) + semantic_roles = { + clean(binding) + for invocation in frame.invocations + for binding in invocation.bindings + if clean(binding) in frame.role_inventory + or clean(binding).lstrip("?") in frame.role_inventory + } + unresolved = tuple(sorted( + role for role in semantic_roles + if role not in bindings and role.lstrip("?") not in bindings + )) + return tuple(bindings[key] for key in sorted(bindings)), unresolved + + @staticmethod + def _residuals(shape: PredicateArgumentShape) -> tuple[AdapterResidual, ...]: + values: list[AdapterResidual] = [] + for feature in shape.features: + if feature.startswith(("voice_", "aspect_", "morph_")): + category, _, value = feature.partition("_") + values.append(AdapterResidual(category, value, False, shape.confidence)) + elif feature == "modal": + values.append(AdapterResidual("modality", "modal_unspecified", False, shape.confidence)) + elif feature == "negated": + values.append(AdapterResidual("polarity", "negative", False, shape.confidence)) + return tuple(values) + + def analyze( + self, + text: str, + *, + maximum_hypotheses: int = 8, + ) -> LanguageAnalysis: + raw_tokens = WORD.findall(str(text)) + values = [value.casefold() for value in raw_tokens] + candidates: list[RelationalProgramHypothesis] = [] + for token_index in self._predicate_indexes(values): + surface = values[token_index] + lemma = self._lemma(surface) + frames = self._lemma_frames.get(lemma, ()) + if not frames: + continue + shape = predicate_argument_shape(text, token_index, self.wordnet) + class_scores = self._sense_class_scores( + text, surface, token_index, lemma, + ) + for frame in frames: + bindings, unresolved = self._bindings(frame, shape) + operator_ids = tuple( + self.basis.canonical_operator_id(value.template_id) + for value in frame.invocations + ) + score = self._frame_score(frame, shape, class_scores) + payload = { + "language": "en", + "surface_sha256": hashlib.sha256(str(text).encode("utf-8")).hexdigest(), + "frame": frame.frame_id, + "operators": operator_ids, + "bindings": [value.as_dict() for value in bindings], + } + candidates.append(RelationalProgramHypothesis( + hypothesis_id="hyp:" + digest(payload)[:28], + language_tag="en", + frame_id=frame.frame_id, + transformation_family_id=frame.transformation_family_id, + predicate_token_index=token_index, + predicate_surface_sha256=hashlib.sha256(surface.encode("utf-8")).hexdigest(), + lemma=lemma, + canonical_operator_ids=operator_ids, + invocations=frame.invocations, + surface_bindings=bindings, + unresolved_roles=unresolved, + residuals=self._residuals(shape), + score=score, + construction_confidence=shape.confidence, + )) + candidates.sort(key=lambda value: (-value.score, value.frame_id)) + hypotheses = tuple(candidates[:max(1, int(maximum_hypotheses))]) + return LanguageAnalysis( + language_tag="en", + surface_sha256=hashlib.sha256(str(text).encode("utf-8")).hexdigest(), + token_count=len(values), + hypotheses=hypotheses, + adapter_id=self.adapter_id, + adapter_semantic_sha256=self.semantic_sha256, + ) + + def instantiate( + self, + hypothesis: RelationalProgramHypothesis, + workspace: EphemeralRelationalWorkspace, + *, + world_id: str | None = None, + ttl_turns: int | None = None, + ) -> InstantiatedAdapterProgram: + if hypothesis.language_tag.casefold() not in { + value.casefold() for value in self.language_tags + }: + raise ValueError("adapter received a hypothesis for another language") + frame = self._frames[hypothesis.frame_id] + self.install_operators(workspace, minimum_support=1) + binding_map = hypothesis.binding_map() + atom_states: list[RelationalState] = [] + generated_variables: list[str] = [] + residual_constraints = tuple( + RelationalConstraint( + kind="language_residual:" + value.feature, + operands=(value.value,), + scope_id=hypothesis.hypothesis_id, + ) + for value in hypothesis.residuals + if value.feature != "polarity" + ) + observed_constraints = tuple( + RelationalConstraint( + kind="language_observed_role", + operands=( + value.role, + ( + "concept:" + value.concept_id + if value.concept_id is not None + else "surface-sha256:" + value.surface_sha256 + ), + value.value_type, + ), + scope_id=hypothesis.hypothesis_id, + ) + for value in hypothesis.surface_bindings + ) + unresolved_constraints = tuple( + RelationalConstraint( + kind="language_unresolved_role", + operands=(value,), + scope_id=hypothesis.hypothesis_id, + ) + for value in hypothesis.unresolved_roles + ) + constraints = ( + residual_constraints + + observed_constraints + + unresolved_constraints + ) + for atom_index, invocation in enumerate(frame.invocations): + template = self.basis.templates[invocation.template_id] + dynamic_templates = [ + value for value in template.arguments if value.dynamic + ] + if len(dynamic_templates) != len(invocation.bindings): + raise RuntimeError("VerbNet operator binding arity drift") + arguments: list[RelationalValue] = [] + for argument_template, source_binding in zip( + dynamic_templates, invocation.bindings, + ): + normalized_binding = clean(source_binding) + if argument_template.dynamic_type == "themrole": + surface = ( + binding_map.get(normalized_binding) + or binding_map.get(normalized_binding.lstrip("?")) + ) + if surface is not None: + arguments.append(workspace.symbol( + surface.surface, + surface.value_type, + world_id=world_id, + concept_id=surface.concept_id, + )) + else: + variable = workspace.new_variable( + normalized_binding.lstrip("?") or "unresolved_role", + "entity", + scope_id=hypothesis.hypothesis_id, + ) + generated_variables.append(variable.variable_id) + arguments.append(RelationalValue.variable(variable)) + else: + arguments.append(workspace.symbol( + normalized_binding or argument_template.dynamic_type or "value", + "value", + world_id=world_id, + )) + atom_states.append(workspace.compose( + self.basis.canonical_operator_id(invocation.template_id), + arguments, + world_id=world_id, + constraints=constraints if atom_index == 0 else (), + ttl_turns=ttl_turns, + provenance=Provenance.local( + f"language-adapter:{self.adapter_id}:{hypothesis.hypothesis_id}:{atom_index}", + ), + )) + if not atom_states: + raise ValueError("selected VerbNet frame has no executable semantic atoms") + result = atom_states[0] + for atom_state in atom_states[1:]: + result = workspace.compose( + "and", + (result, atom_state), + world_id=world_id, + ttl_turns=ttl_turns, + provenance=Provenance.local( + f"language-adapter-compose:{hypothesis.hypothesis_id}", + parent_state_ids=(result.state_id, atom_state.state_id), + ), + ) + if any( + value.feature == "polarity" and value.value == "negative" + for value in hypothesis.residuals + ): + result = workspace.compose( + "negate", (result,), world_id=world_id, ttl_turns=ttl_turns, + provenance=Provenance.local( + f"language-adapter-negation:{hypothesis.hypothesis_id}", + parent_state_ids=(result.state_id,), + ), + ) + return InstantiatedAdapterProgram( + hypothesis=hypothesis, + atom_state_ids=tuple(value.state_id for value in atom_states), + result_state=result, + generated_variable_ids=tuple(generated_variables), + ) + + +class CaseMarkedVerbNetAdapter(LanguageAdapter): + """Bounded SOV/case-particle adapter used to test non-English parity. + + Lexical entries point directly to structural frame ids. Runtime analysis + never translates through English. The class is language-generic; Japanese + romanized case particles are one initial configuration. + """ + + def __init__( + self, + basis: VerbNetOperatorBasis, + *, + language_tag: str, + verb_lexicon: Mapping[str, Sequence[str]], + case_slots: Mapping[str, str], + concept_lexicon: Mapping[str, str] | None = None, + adapter_name: str = "case-marked-adapter-b", + ) -> None: + self.basis = basis + self.language_tags = (str(language_tag),) + self.adapter_id = ( + "oracle-stage5k3-" + clean(adapter_name) + "-v2" + ) + self._frames = {frame.frame_id: frame for frame in basis.frames} + self.verb_lexicon = { + clean(key): tuple(str(value) for value in values) + for key, values in verb_lexicon.items() + } + self.case_slots = { + clean(key): str(value) for key, value in case_slots.items() + } + self.concept_lexicon = { + clean(key): str(value) + for key, value in (concept_lexicon or {}).items() + } + for frame_ids in self.verb_lexicon.values(): + for frame_id in frame_ids: + if frame_id not in self._frames: + raise KeyError(f"case adapter names unknown frame {frame_id}") + self.semantic_sha256 = digest({ + "adapter_id": self.adapter_id, + "language_tag": language_tag, + "basis_source": basis.source_sha256, + "verb_lexicon": self.verb_lexicon, + "case_slots": self.case_slots, + "concept_lexicon": self.concept_lexicon, + "observation_constraints": "stage5k3-v2", + }) + + def install_operators( + self, + workspace: EphemeralRelationalWorkspace, + *, + minimum_support: int = 1, + ) -> tuple[str, ...]: + return self.basis.register_supported_operators( + workspace.canonical, minimum_support=minimum_support, + ) + + @staticmethod + def _tokens(text: str) -> tuple[str, ...]: + return tuple( + clean(value.strip(".,!?;:()[]{}")) + for value in str(text).split() + if value.strip(".,!?;:()[]{}") + ) + + def _case_fillers( + self, + tokens: Sequence[str], + ) -> dict[str, tuple[str, int]]: + values: dict[str, tuple[str, int]] = {} + for index, token in enumerate(tokens): + slot = self.case_slots.get(token) + if slot is None or index == 0: + continue + values[slot] = (tokens[index - 1], index - 1) + return values + + @staticmethod + def _surface_for_slot( + frame_slot: str, + fillers: Mapping[str, tuple[str, int]], + ) -> tuple[str, int] | None: + if frame_slot in fillers: + return fillers[frame_slot] + if frame_slot.startswith("prep:"): + return ( + fillers.get(frame_slot) + or fillers.get("prep") + or fillers.get("goal") + or fillers.get("source") + ) + if frame_slot.startswith("object:"): + return fillers.get(frame_slot) or fillers.get("indirect_object") + return None + + def analyze( + self, + text: str, + *, + maximum_hypotheses: int = 8, + ) -> LanguageAnalysis: + tokens = self._tokens(text) + fillers = self._case_fillers(tokens) + candidates: list[RelationalProgramHypothesis] = [] + for predicate_index, token in enumerate(tokens): + frame_ids = self.verb_lexicon.get(token, ()) + for frame_id in frame_ids: + frame = self._frames[frame_id] + bindings: dict[str, SurfaceBinding] = {} + for frame_slot, role in frame.surface_slots: + value = self._surface_for_slot(frame_slot, fillers) + if value is None or role in bindings: + continue + surface, token_index = value + bindings[role] = SurfaceBinding.create( + self.language_tags[0], + role, + surface, + concept_id=self.concept_lexicon.get(clean(surface)), + token_start=token_index, + token_stop=token_index + 1, + confidence=0.98, + ) + semantic_roles = { + clean(binding) + for invocation in frame.invocations + for binding in invocation.bindings + if clean(binding) in frame.role_inventory + or clean(binding).lstrip("?") in frame.role_inventory + } + unresolved = tuple(sorted( + role for role in semantic_roles + if role not in bindings and role.lstrip("?") not in bindings + )) + operator_ids = tuple( + self.basis.canonical_operator_id(value.template_id) + for value in frame.invocations + ) + surface_bindings = tuple( + bindings[key] for key in sorted(bindings) + ) + coverage = len(surface_bindings) / max(1, len(frame.surface_slots)) + payload = { + "language": self.language_tags[0], + "surface_sha256": hashlib.sha256( + str(text).encode("utf-8") + ).hexdigest(), + "frame": frame.frame_id, + "operators": operator_ids, + "bindings": [value.as_dict() for value in surface_bindings], + } + candidates.append(RelationalProgramHypothesis( + hypothesis_id="hyp:" + digest(payload)[:28], + language_tag=self.language_tags[0], + frame_id=frame.frame_id, + transformation_family_id=frame.transformation_family_id, + predicate_token_index=predicate_index, + predicate_surface_sha256=hashlib.sha256( + token.encode("utf-8") + ).hexdigest(), + lemma=token, + canonical_operator_ids=operator_ids, + invocations=frame.invocations, + surface_bindings=surface_bindings, + unresolved_roles=unresolved, + residuals=( + AdapterResidual( + "word_order", "sov", True, 1.0, + ), + AdapterResidual( + "case_marking", "particle", True, 1.0, + ), + ), + score=min(1.0, 0.35 + 0.65 * coverage), + construction_confidence=0.98, + )) + candidates.sort(key=lambda value: (-value.score, value.frame_id)) + return LanguageAnalysis( + language_tag=self.language_tags[0], + surface_sha256=hashlib.sha256(str(text).encode("utf-8")).hexdigest(), + token_count=len(tokens), + hypotheses=tuple( + candidates[:max(1, int(maximum_hypotheses))] + ), + adapter_id=self.adapter_id, + adapter_semantic_sha256=self.semantic_sha256, + ) + + def instantiate( + self, + hypothesis: RelationalProgramHypothesis, + workspace: EphemeralRelationalWorkspace, + *, + world_id: str | None = None, + ttl_turns: int | None = None, + ) -> InstantiatedAdapterProgram: + # The execution contract is language independent. + return EnglishRelationalAdapter.instantiate( + self, + hypothesis, + workspace, + world_id=world_id, + ttl_turns=ttl_turns, + ) + + +def re_split(value: str) -> tuple[str, ...]: + result = value.replace("|", "_").replace("/", "_").split("_") + return tuple(item for item in result if item) + + +__all__ = [ + "AdapterResidual", + "CaseMarkedVerbNetAdapter", + "EnglishRelationalAdapter", + "InstantiatedAdapterProgram", + "LanguageAdapter", + "LanguageAdapterRegistry", + "LanguageAnalysis", + "RelationalProgramHypothesis", + "SurfaceBinding", +] diff --git a/root/wander-train/stage5k43_causal_relational_native_generalist_agent.py b/root/wander-train/stage5k43_causal_relational_native_generalist_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..a68f3f62e9f87adadeeb79c8f85a8294b484dad6 --- /dev/null +++ b/root/wander-train/stage5k43_causal_relational_native_generalist_agent.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""K43 causal-graph and epistemic-admission repairs from immutable K42. + +K42 remains rejected. This successor changes no corpus row, specialist, +checkpoint, learned carrier, or routing cell. It repairs three deterministic +interfaces exposed by K42's committed failures: + +* sentence-internal ``because`` is compiled as a typed causal edge before a + nested copula can consume the sentence; +* remembered effects retain a multiword subject during realization; and +* a causal source may enter proof only with K41's rare exact-anchor + specificity certificate, not merely generic semantic overlap. +""" + +from __future__ import annotations + +from pathlib import Path +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest, tokens +from stage5k39_semantic_indexed_native_generalist_agent import ( + SemanticIndexedNativeGeneralistAgent, +) +from stage5k41_relational_specificity_native_generalist_agent import ( + K41ProductivePropositionCompiler, + RelationalSpecificityNativeGeneralistAgent, +) + + +SCHEMA = "oracle-stage5k43-causal-relational-native-generalist-agent-v1" + + +class K43CausalRelationalPropositionCompiler(K41ProductivePropositionCompiler): + """Compile an internal causal connective before nested predicate search.""" + + def compile_row( + self, + evidence_row: Mapping[str, Any], + domain_ids: Sequence[str] = (), + ): + evidence = clean( + evidence_row.get("fact", evidence_row.get("text", "")) + ).rstrip(".!?") + causal = re.match(r"^(.+?)\s+because\s+(.+)$", evidence, re.IGNORECASE) + if causal is None: + return super().compile_row(evidence_row, domain_ids) + result_surface = clean(causal.group(1)) + support_surface = clean(causal.group(2)) + result = self._parse_simple( + result_surface, + evidence_row, + domain_ids, + clause_type="causal_result", + extra_qualifiers=(("because", support_surface),), + base_offset=causal.start(1), + ) + support = self._parse_simple( + support_surface, + evidence_row, + domain_ids, + clause_type="causal_support", + base_offset=causal.start(2), + ) + values = tuple(value for value in (result, support) if value is not None) + if not values: + return super().compile_row(evidence_row, domain_ids) + return tuple({value.provenance_sha256: value for value in values}.values()) + + +class CausalRelationalNativeGeneralistAgent(RelationalSpecificityNativeGeneralistAgent): + """K41 four-cell runtime with K42-localized causal repairs.""" + + def __init__( + self, + specialist_checkpoint: Path, + knowledge_checkpoint: Path, + address_index_path: Path, + relational_index_path: Path, + concept_index_path: Path, + substrate_path: Path, + **kwargs: Any, + ) -> None: + kwargs.setdefault("agent_id", "oracle-stage5k43-causal-relational-agent") + super().__init__( + specialist_checkpoint, + knowledge_checkpoint, + address_index_path, + relational_index_path, + concept_index_path, + substrate_path, + **kwargs, + ) + self.proposition_compiler = K43CausalRelationalPropositionCompiler( + self.knowledge_basis.concept_index, + ) + + @classmethod + def _routing_goal( + cls, + text: str, + context: Mapping[str, Any], + ) -> tuple[tuple[str, ...], ...]: + intent = cls._intent_lattice(text, context).get("selected_intent_family") + if intent == "causal_query" and not context.get("evidence"): + # Unsupported causal judgment is conjunctive: causal structure + # must be recognized and epistemic authority must be present. + # A single mechanism specialist is not equivalent to that pair. + return (("causal",), ("epistemic",)) + return super()._routing_goal(text, context) + + @staticmethod + def _retrieval_gate( + intent_family: str, + evidence: Sequence[Mapping[str, Any]], + context: Mapping[str, Any], + ) -> tuple[bool, str]: + if intent_family != "causal_query": + return SemanticIndexedNativeGeneralistAgent._retrieval_gate( + intent_family, evidence, context, + ) + if any(context.get(key) for key in ("evidence", "training_corpus", "sensor_stream")): + return False, "caller_supplied_bounded_context_has_precedence" + if not evidence: + return False, "causal_query_has_no_selected_domain_evidence" + best = evidence[0] + causal_surface = bool(re.search( + r"\b(?:because|therefore|thus|hence|caus(?:e|es|ed)|results?\s+(?:from|in)|due\s+to|leads?\s+to|so\s+that)\b", + str(best.get("text", "")).casefold(), + )) + specificity = bool( + best.get("relational_specificity_admitted") + and int(best.get("relational_exact_anchor_count", 0)) >= 2 + and float(best.get("relational_exact_information", 0.0)) >= 12.0 + and float(best.get("relational_exact_information_margin", 0.0)) >= 2.0 + ) + if causal_surface and specificity: + return True, "causal_source_has_specific_exact_relational_certificate" + return False, "causal_source_lacks_specific_exact_relational_certificate" + + def _handle_event(self, session_id: str, text: str, turn_id: str): + if "why" in set(tokens(text)): + payloads, memory_audit = self._retrieve( + session_id, + {"situation": "causal_chain"}, + turn_id, + ) + match = next( + (value for value in payloads if value.get("situation") == "causal_chain"), + None, + ) + if match is not None: + effect = clean(match["result"]) + words = effect.split() + predicate = "become" + subject = words[0] if words else "outcome" + complement: str | None = " ".join(words[1:]) or "the resulting state" + if "became" in words: + pivot = words.index("became") + subject = " ".join(words[:pivot]) or "outcome" + predicate = "become" + complement = " ".join(words[pivot + 1:]) or "the resulting state" + elif words: + resolved = self.proposition_compiler._lemma_profile(words[-1]) + if resolved is not None and len(words) >= 2: + subject = " ".join(words[:-1]) + predicate = str(resolved[0]) + complement = None + return HandlerResult( + "causal_explanation", + "answered", + (("effect", effect),), + (("cause", match["source"]), ("intermediate", match["field"])), + (SemanticClause( + "The " + subject, + predicate, + complement=complement, + qualifiers=( + ("because", "the " + match["field"]), + ("after", "the " + match["source"]), + ), + tense="past", + ),), + (("memory_snapshot", memory_audit["snapshot_digest"]),), + ( + "retrieve_causal_chain", + "retain_multiword_effect_subject", + "release_ordered_links", + "construct_explanation", + ), + memory_audit=memory_audit, + ) + return super()._handle_event(session_id, text, turn_id) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ablate_knowledge_cells: Sequence[str] = (), + ) -> dict[str, Any]: + output = super().answer( + session_id, + text, + context, + ablate_specialists=ablate_specialists, + ablate_knowledge_cells=ablate_knowledge_cells, + ) + causal_request = bool(set(tokens(text)) & {"why", "cause", "caused", "causal"}) + evidence = (output.get("knowledge_lattice") or {}).get("retrieved_evidence", ()) + best = evidence[0] if evidence else {} + output["causal_specificity_gate"] = { + "invoked": causal_request, + "retrieval_admitted_to_proof": bool( + (output.get("knowledge_lattice") or {}).get("retrieval_admitted_to_proof") + ), + "best_text_sha256": best.get("text_sha256"), + "specific_exact_certificate": bool(best.get("relational_specificity_admitted")), + "exact_anchor_count": int(best.get("relational_exact_anchor_count", 0)), + "exact_information": float(best.get("relational_exact_information", 0.0)), + "exact_information_margin": float(best.get("relational_exact_information_margin", 0.0)), + "answer_or_gold_labels_used": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["schema_version"] = SCHEMA + output["runtime"]["causal_relation_compiler_model_calls"] = 0 + output["runtime"]["causal_relation_compiler_gradient_steps"] = 0 + output["runtime"]["causal_specificity_gate_model_calls"] = 0 + output["runtime"]["causal_specificity_gate_gradient_steps"] = 0 + output["runtime"]["multiword_effect_realizer_model_calls"] = 0 + output["runtime"]["multiword_effect_realizer_gradient_steps"] = 0 + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = [ + "CausalRelationalNativeGeneralistAgent", + "K43CausalRelationalPropositionCompiler", +] diff --git a/root/wander-train/stage5k57_ablatable_clause_algebra_agent.py b/root/wander-train/stage5k57_ablatable_clause_algebra_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..855a4afa96fdddac7accae59f43cc5d593b313af --- /dev/null +++ b/root/wander-train/stage5k57_ablatable_clause_algebra_agent.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""K57 successor: enforce destructive controls at the K53/K55 boundary.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, digest +from stage5k51_restart_safe_ephemeral_agent import IndexedEphemeralEventJournal +from stage5k55_article_aware_clause_algebra_agent import ArticleAwareClauseAlgebraAgent + + +SCHEMA = "oracle-stage5k57-ablatable-clause-algebra-agent-v1" + + +class AblatableClauseAlgebraAgent(ArticleAwareClauseAlgebraAgent): + def __init__( + self, + *args: Any, + ephemeral_journal_path: Path | None = None, + **kwargs: Any, + ) -> None: + substrate_path = Path(args[5] if len(args) >= 6 else kwargs["substrate_path"]) + journal_path = ephemeral_journal_path or substrate_path.with_suffix( + ".k57.ephemeral.sqlite3" + ) + kwargs.setdefault("agent_id", "oracle-stage5k57-ablatable-clause-agent") + super().__init__(*args, ephemeral_journal_path=journal_path, **kwargs) + state = hashlib.sha256() + for name in ( + "stage5k51_restart_safe_ephemeral_agent.py", + "stage5k53_native_clause_algebra_agent.py", + "stage5k55_article_aware_clause_algebra_agent.py", + Path(__file__).name, + ): + payload = Path(__file__).with_name(name).read_bytes() + state.update(len(payload).to_bytes(8, "big")) + state.update(payload) + self._k53_lineage_sha256 = state.hexdigest() + self.ephemeral_journal = IndexedEphemeralEventJournal( + journal_path, source_sha256=self._k53_lineage_sha256, + ) + + def _handle_event(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + if self._active_intent.startswith("ephemeral_") and self._k45_ablate_workspace: + return self._ablation("native_clause_workspace") + if ( + self._active_intent == "ephemeral_rule" + and self._k45_ablate_composition + and self._quantified_rule(text) is not None + ): + return self._ablation("compose_quantified_rule") + return super()._handle_event(session_id, text, turn_id) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ablate_knowledge_cells: Sequence[str] = (), + ablate_ephemeral_workspace: bool = False, + ablate_operator_composition: bool = False, + ) -> dict[str, Any]: + output = super().answer( + session_id, text, context, + ablate_specialists=ablate_specialists, + ablate_knowledge_cells=ablate_knowledge_cells, + ablate_ephemeral_workspace=ablate_ephemeral_workspace, + ablate_operator_composition=ablate_operator_composition, + ) + output["native_clause_algebra"]["workspace_ablation_enforced"] = True + output["native_clause_algebra"]["quantified_composition_ablation_enforced"] = True + output["runtime"]["clause_ablation_model_calls"] = 0 + output["runtime"]["clause_ablation_gradient_steps"] = 0 + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["AblatableClauseAlgebraAgent"] diff --git a/root/wander-train/stage5k76_relational_graph_transduction_agent.py b/root/wander-train/stage5k76_relational_graph_transduction_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..478f3cb3f6b106fb86f7e77c567576e28c93522e --- /dev/null +++ b/root/wander-train/stage5k76_relational_graph_transduction_agent.py @@ -0,0 +1,2442 @@ +#!/usr/bin/env python3 +"""K76 native utterance-to-relational-graph transduction. + +K73 demonstrated that surface-equivalence rewrites can transfer a bounded set +of English constructions, but K75 rejected semantic freedom: novel speech +acts were still liable to be parsed as asserted events and discourse updates +were not represented by one common object. K76 leaves the frozen K73 lineage +untouched and introduces that missing object. + +The transducer is deliberately not a language model. It uses closed-class +topology, the frozen K63 dictionary, typed variables, graph unification, and +the existing HRR relational workspace. Surface text is used to construct a +whole utterance graph; answers are released by the inherited native inverse +grammar after graph operations have completed. +""" + +from __future__ import annotations + +from contextvars import ContextVar +from dataclasses import dataclass, field, replace +import hashlib +from pathlib import Path +import re +import threading +from typing import Any, Iterable, Mapping, Sequence + +from stage5j1_native_contextual_sense_lattice import WORD +from stage5j2_native_english_construction_algebra import ( + AUXILIARIES, + DETERMINERS, + MODALS, + PREPOSITIONS, +) +from stage5k1_generative_relational_workspace import RelationalValue +from stage5k15_native_mixed_agent import ( + HandlerResult, + SemanticClause, + clean, + digest, +) +from stage5k45_ephemeral_compositional_language_agent import RelationFact, _strip_article +from stage5k63_contextual_frame_induction_agent import FrameArgument, InducedFrame +from stage5k73_construction_equivalence_agent import ConstructionEquivalenceAgent + + +SCHEMA = "oracle-stage5k76-relational-graph-transduction-agent-v1" + + +def _words(text: str) -> tuple[str, ...]: + return tuple(value.casefold() for value in WORD.findall(str(text))) + + +def _surface(values: Sequence[str]) -> str: + result = clean(" ".join(values)).strip(" .?!,:;\"'") + return _strip_article(result) + + +def _singular(value: str) -> str: + word = clean(value).casefold() + if word.endswith("ies") and len(word) > 3: + return word[:-3] + "y" + if word.endswith("ses") and len(word) > 4: + return word[:-2] + if word.endswith("s") and not word.endswith(("ss", "us")): + return word[:-1] + return word + + +@dataclass(frozen=True) +class GraphAtom: + """A typed proposition; variables are strings beginning with ``$``.""" + + subject: str + predicate: str + object_value: str + world: str = "actual" + polarity: str = "positive" + qualifiers: tuple[tuple[str, str], ...] = () + + def as_dict(self) -> dict[str, Any]: + return { + "subject": self.subject, + "predicate": self.predicate, + "object": self.object_value, + "world": self.world, + "polarity": self.polarity, + "qualifiers": [list(value) for value in self.qualifiers], + } + + +@dataclass(frozen=True) +class GraphRule: + rule_id: str + antecedents: tuple[GraphAtom, ...] + consequents: tuple[GraphAtom, ...] + world: str = "actual" + exception: GraphAtom | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "rule_id": self.rule_id, + "antecedents": [value.as_dict() for value in self.antecedents], + "consequents": [value.as_dict() for value in self.consequents], + "world": self.world, + "exception": None if self.exception is None else self.exception.as_dict(), + } + + +@dataclass(frozen=True) +class GraphDefinition: + term: str + kind: str + properties: tuple[str, ...] + shape: str | None + relation: str | None + relation_object: str | None + definition_state_id: str | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "term": self.term, + "kind": self.kind, + "properties": list(self.properties), + "shape": self.shape, + "relation": self.relation, + "relation_object": self.relation_object, + "definition_state_id": self.definition_state_id, + } + + +@dataclass +class StoredGraphFact: + atom: GraphAtom + state_id: str + active: bool = True + inferred: bool = False + supports: tuple[str, ...] = () + + +@dataclass +class GraphDiscourse: + definitions: dict[str, GraphDefinition] = field(default_factory=dict) + instances: dict[str, str] = field(default_factory=dict) + facts: list[StoredGraphFact] = field(default_factory=list) + rules: list[GraphRule] = field(default_factory=list) + causal_edges: list[tuple[str, str]] = field(default_factory=list) + last_entity: str | None = None + last_event_subject: str | None = None + last_event_object: str | None = None + last_event_destination: str | None = None + last_property_entity: str | None = None + current_counterfactual_entity: str | None = None + current_hypothetical_rule: GraphRule | None = None + + +@dataclass(frozen=True) +class UtteranceGraph: + speech_act: str + atoms: tuple[GraphAtom, ...] = () + payload: Mapping[str, Any] = field(default_factory=dict) + topology: tuple[str, ...] = () + confidence: float = 0.0 + consumes_as_graph: bool = False + + def as_dict(self) -> dict[str, Any]: + serial_payload: dict[str, Any] = {} + for key, value in self.payload.items(): + if isinstance(value, InducedFrame): + serial_payload[key] = value.as_dict() + elif isinstance(value, GraphRule): + serial_payload[key] = value.as_dict() + elif isinstance(value, GraphDefinition): + serial_payload[key] = value.as_dict() + elif isinstance(value, GraphAtom): + serial_payload[key] = value.as_dict() + else: + serial_payload[key] = value + return { + "speech_act": self.speech_act, + "atoms": [value.as_dict() for value in self.atoms], + "payload": serial_payload, + "topology": list(self.topology), + "confidence": self.confidence, + "consumes_as_graph": self.consumes_as_graph, + } + + +class ClosedClassGraphTransducer: + """Map whole English constructions to typed graph operations. + + This is a small construction algebra, not an inventory of answer strings. + The decisive signals are clause order, auxiliaries, complementizers, + quantifiers, prepositions, modality, correction/world markers, and the + locations of predicates supplied by the frozen dictionary. + """ + + SHAPES = { + "triangular": "triangle", + "triangle": "triangle", + "octagonal": "octagon", + "octagon": "octagon", + "circular": "circle", + "round": "circle", + "square": "square", + "rectangular": "rectangle", + "hexagonal": "hexagon", + "pentagonal": "pentagon", + "spherical": "sphere", + } + GREETINGS = frozenset({"hello", "hi", "greetings", "morning", "afternoon", "evening"}) + MUTATIONS = frozenset({"alter", "change", "delete", "insert", "modify", "overwrite", "remove", "update", "write"}) + TOOL_NOUNS = frozenset({ + "catalog", "cataloged", "database", "dataset", "table", "record", + "earthquake", "earthquakes", + }) + WH = frozenset({"what", "which", "who", "where", "when", "why", "how"}) + IMPERATIVE_CUES = frozenset({ + "answer", "calculate", "classify", "compare", "connect", "contrast", + "count", "define", "describe", "enumerate", "explain", "give", + "identify", "juxtapose", "list", "locate", "name", "report", + "relate", "restate", "retrieve", "state", "summarize", "trace", "use", "write", + }) + + def __init__(self, owner: "RelationalGraphTransductionAgent") -> None: + self.owner = owner + + def lemma(self, surface: str) -> str: + value = clean(surface).casefold() + lemma, listed = self.owner.dictionary.lemma(value) + if listed: + return lemma + if value.endswith("ied") and len(value) > 4: + return value[:-3] + "y" + if value.endswith("ing") and len(value) > 5: + stem = value[:-3] + if len(stem) >= 2 and stem[-1] == stem[-2]: + stem = stem[:-1] + return stem + if value.endswith("ed") and len(value) > 4: + stem = value[:-2] + if len(stem) >= 2 and stem[-1] == stem[-2]: + stem = stem[:-1] + return stem + if value.endswith("s") and len(value) > 3: + return _singular(value) + return lemma or value + + @staticmethod + def _quoted_term(text: str) -> str | None: + match = re.search(r"['\"]([A-Za-z][A-Za-z0-9_-]*)['\"]", text) + return match.group(1).casefold() if match else None + + def _definition(self, text: str) -> GraphDefinition | None: + raw = tuple(WORD.findall(text)) + values = tuple(value.casefold() for value in raw) + connectors = [ + index for index, value in enumerate(values) + if value in {"denote", "denotes", "mean", "means"} + ] + if not connectors: + return None + connector = connectors[0] + term = self._quoted_term(text) + if term is None and values and values[0] == "let" and connector >= 2: + term = values[1] + if term is None and connector >= 1: + term = values[connector - 1] + if not term: + return None + description = list(values[connector + 1:]) + while description and description[0] in {"as", "a", "an", "the"}: + description.pop(0) + if not description: + return None + relative = len(description) + relation: str | None = None + relation_object: str | None = None + used_for = None + for index in range(len(description) - 1): + if description[index:index + 2] == ["used", "for"]: + used_for = index + relative = min(relative, index) + break + that_index = next(( + index for index, value in enumerate(description) if value in {"that", "which"} + ), None) + if that_index is not None: + relative = min(relative, that_index) + nominal = description[:relative] + kind = nominal[-1] if nominal else "thing" + modifiers = nominal[:-1] + shape = next((self.SHAPES[value] for value in modifiers if value in self.SHAPES), None) + properties = tuple(value for value in modifiers if value not in self.SHAPES) + if used_for is not None and used_for + 2 < len(description): + relation = self.lemma(description[used_for + 2]) + relation_object = _surface(description[used_for + 3:]) or "its stated purpose" + elif that_index is not None and that_index + 1 < len(description): + relation = self.lemma(description[that_index + 1]) + relation_object = _surface(description[that_index + 2:]) or "its stated object" + return GraphDefinition(term, kind, properties, shape, relation, relation_object) + + @staticmethod + def _known_concept(values: Sequence[str], discourse: GraphDiscourse) -> str | None: + concepts = set(discourse.definitions) + for value in reversed(values): + candidate = _singular(value) + if candidate in concepts: + return candidate + return None + + def _membership(self, text: str, discourse: GraphDiscourse) -> tuple[str, str] | None: + raw = tuple(WORD.findall(text)) + values = tuple(value.casefold() for value in raw) + concept = self._known_concept(values, discourse) + if concept is None: + return None + entity: str | None = None + if values and values[0] == "count" and "among" in values: + among = values.index("among") + entity = _surface(raw[1:among]) + elif "qualifies" in values and "as" in values: + q = values.index("qualifies") + begin = 1 if values and values[0] == "suppose" else 0 + entity = _surface(raw[begin:q]) + elif "is" in values: + index = values.index("is") + entity = _surface(raw[:index]) + return (entity, concept) if entity else None + + @staticmethod + def _instance_query(text: str, discourse: GraphDiscourse) -> str | None: + lower = clean(text).casefold() + for entity in reversed(tuple(discourse.instances)): + if re.search(rf"\b{re.escape(entity)}(?:'s)?\b", lower): + return entity + if discourse.last_entity and any( + cue in lower for cue in ( + "what can you say", "what commits", "commits us", "enumerate", + "infer", "true about", + ) + ): + return discourse.last_entity + return None + + def _simple_clause(self, text: str, *, world: str = "actual") -> GraphAtom | None: + value = clean(text).strip(" .?!,:;") + token_matches = list(WORD.finditer(value)) + raw = [match.group(0) for match in token_matches] + words = [item.casefold() for item in raw] + if len(words) < 2: + return None + stripped_prefix = 0 + while words and words[0] in DETERMINERS: + raw.pop(0); words.pop(0); stripped_prefix += 1 + copula = next((index for index, item in enumerate(words) if item in {"is", "are", "was", "were"}), None) + if copula is not None and 0 < copula < len(words) - 1: + subject = _surface(raw[:copula]) + complement = _surface(raw[copula + 1:]) + if words[copula + 1] in {"a", "an"}: + type_index = copula + 2 + relation_index = next(( + index for index in range(type_index + 1, len(words)) + if words[index] in {"inside", "within", "beneath", "above", "below"} + ), None) + qualifiers: tuple[tuple[str, str], ...] = () + if relation_index is not None: + qualifiers = (( + words[relation_index], + _surface(raw[relation_index + 1:]), + ),) + return GraphAtom( + subject, "is_a", _singular(words[type_index]), world, + qualifiers=qualifiers, + ) + return GraphAtom(subject, "property", complement, world) + inflected_candidates = [ + index for index in range(1, len(words)) + if ( + words[index].endswith(("ed", "ied", "en", "ing")) + or ( + words[index].endswith("s") + and self.owner.dictionary.is_listed_verb(_singular(words[index])) + ) + ) + and words[index] not in AUXILIARIES | MODALS + ] + predicate_index = inflected_candidates[0] if inflected_candidates else self.owner.frame_inducer._predicate_index(words, start=1) + if predicate_index is None: + # Unlisted predicates are licensed by the slot between a nominal + # subject and a following nominal/prepositional complement. + predicate_index = 1 if len(words) >= 2 else None + if predicate_index is None or predicate_index == 0: + return None + subject = _surface(raw[:predicate_index]) + predicate = self.lemma(words[predicate_index]) + # Tokenization is used for topology, but the object surface is sliced + # from the original clause. That preserves meaningful punctuation + # such as the hyphen in ``ember-seed`` through graph binding/release. + absolute_predicate_index = stripped_prefix + predicate_index + object_value = _strip_article( + clean(value[token_matches[absolute_predicate_index].end():]).strip(" .?!,:;") + ) or "event" + return GraphAtom(subject, predicate, object_value, world) + + def _causal(self, text: str) -> tuple[GraphAtom, GraphAtom] | None: + value = clean(text).strip(" .?!") + match = re.match(r"^(.+?)\s+because\s+(.+)$", value, re.IGNORECASE) + if match is None: + return None + effect = self._simple_clause(match.group(1)) + cause = self._simple_clause(match.group(2)) + return (cause, effect) if cause and effect else None + + def _initial_adjunct_frame(self, text: str) -> InducedFrame | None: + original = clean(text) + raw = list(WORD.findall(original)) + values = [value.casefold() for value in raw] + if not values or values[0] not in PREPOSITIONS: + return None + prep = values[0] + adjunct_end: int | None = None + if "," in original: + prefix = original.split(",", 1)[0] + adjunct_end = len(WORD.findall(prefix)) + else: + # A leading adjunct ends at the next determiner that begins a + # finite clause: ``At dusk | the navigator ...``. + adjunct_end = next(( + index for index in range(2, len(values)) + if values[index] in DETERMINERS + ), None) + if adjunct_end is None or adjunct_end >= len(values) - 1: + return None + adjunct_value = _surface(raw[1:adjunct_end]) + main = clean(" ".join(raw[adjunct_end:])) + "." + frame = self.owner.frame_inducer.analyze_assertion(main) + if frame is None or not adjunct_value: + return None + normalized_lemma = self.lemma(frame.predicate_surface) + arguments = tuple(frame.arguments) + ( + FrameArgument("prep:" + prep, adjunct_value, 0, marker=prep), + ) + signature = f"{frame.voice}|" + ",".join(value.role for value in arguments) + f"|clause:{frame.clause_kind or 'none'}" + return replace( + frame, lemma=normalized_lemma, + arguments=arguments, structural_signature=signature, + ) + + def _frame_query( + self, + text: str, + discourse: GraphDiscourse, + known_frames: Sequence[Mapping[str, Any]] = (), + ) -> Mapping[str, Any] | None: + values = _words(text) + wh_index = next(( + index for index, value in enumerate(values) + if value in {"which", "what", "who"} + ), None) + if not values or wh_index is None: + return None + values = values[wh_index:] + target = "subject" if values[0] == "who" else "direct_object" + known = [ + fact.atom.predicate for fact in discourse.facts + if fact.active and fact.atom.predicate not in {"is_a", "property"} + ] + known.extend(str(value.get("lemma", "")) for value in known_frames) + predicate = next(( + self.lemma(value) for value in values + if self.lemma(value) in known + ), None) + if predicate is None: + return None + anchors = tuple( + _singular(value) for value in values + if value not in self.WH | AUXILIARIES | MODALS | PREPOSITIONS | DETERMINERS + and self.lemma(value) != predicate + and value not in {"thing", "reconstructing", "conversation", "after"} + ) + return {"predicate": predicate, "target_role": target, "anchors": anchors} + + @staticmethod + def _number_value(values: Sequence[str]) -> int | None: + small = { + "zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, + "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, + "ten": 10, "eleven": 11, "twelve": 12, "thirteen": 13, + "fourteen": 14, "fifteen": 15, "sixteen": 16, "seventeen": 17, + "eighteen": 18, "nineteen": 19, + } + tens = { + "twenty": 20, "thirty": 30, "forty": 40, "fifty": 50, + "sixty": 60, "seventy": 70, "eighty": 80, "ninety": 90, + } + total = 0 + current = 0 + consumed = False + for value in values: + if value.isdigit(): + current += int(value); consumed = True + elif value in small: + current += small[value]; consumed = True + elif value in tens: + current += tens[value]; consumed = True + elif value == "hundred" and consumed: + current = max(1, current) * 100 + elif value == "thousand" and consumed: + total += max(1, current) * 1000; current = 0 + elif consumed: + break + return total + current if consumed else None + + @classmethod + def _number_spans(cls, values: Sequence[str]) -> tuple[int, ...]: + vocabulary = { + "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", + "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", + "eighteen", "nineteen", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", + "eighty", "ninety", "hundred", "thousand", + } + result: list[int] = [] + index = 0 + while index < len(values): + if not (values[index].isdigit() or values[index] in vocabulary): + index += 1; continue + end = index + 1 + while end < len(values): + if values[end].isdigit() or values[end] in vocabulary: + end += 1 + continue + # ``and`` is internal to a single English numeral only after + # a scale word (one hundred and five); otherwise it separates + # operands (twenty-seven and sixteen). + if ( + values[end] == "and" + and "hundred" in values[index:end] + and end + 1 < len(values) + and (values[end + 1].isdigit() or values[end + 1] in vocabulary) + ): + end += 1 + continue + break + number = cls._number_value([value for value in values[index:end] if value != "and"]) + if number is not None: + result.append(number) + index = end + return tuple(result) + + @staticmethod + def _rule_id(antecedents: Sequence[GraphAtom], consequents: Sequence[GraphAtom]) -> str: + payload = [value.as_dict() for value in (*antecedents, *consequents)] + return "graph-rule:" + digest(payload)[:24] + + def _universal_rule(self, text: str) -> GraphRule | None: + lower = clean(text).casefold().strip(" .?!") + # Quantified actor + shared object variable. + match = re.match( + r"^whoever\s+is\s+(?:a|an)\s+([a-z][a-z-]*)\s+and\s+" + r"([a-z][a-z-]*)\s+(?:a|an|the)\s+([a-z][a-z-]*)\s+" + r"must\s+([a-z][a-z-]*)\s+(?:that\s+same|the\s+same|that|the)\s+\3$", + lower, + ) + if match: + entity_type, antecedent, object_type, consequent = match.groups() + before = ( + GraphAtom("$x", "is_a", entity_type), + GraphAtom("$x", self.lemma(antecedent), "$y", qualifiers=(("object_type", object_type),)), + ) + after = (GraphAtom("$x", self.lemma(consequent), "$y"),) + return GraphRule(self._rule_id(before, after), before, after) + + # Every TYPE inside LOCATION becomes PROPERTY. + match = re.match( + r"^every\s+([a-z][a-z-]*)\s+(inside|within|beneath|above|below|north\s+of|south\s+of)\s+" + r"(?:a|an|the)\s+(.+?)\s+(?:becomes?|is)\s+([a-z][a-z-]*)$", + lower, + ) + if match: + entity_type, relation, location, quality = match.groups() + relation = relation.replace(" ", "_") + before = ( + GraphAtom("$x", "is_a", entity_type), + GraphAtom("$x", relation, _strip_article(location)), + ) + after = (GraphAtom("$x", "property", quality),) + return GraphRule(self._rule_id(before, after), before, after) + + # Every PROPERTY TYPE PREDICATE OBJECT. + match = re.match( + r"^every\s+([a-z][a-z-]*)\s+([a-z][a-z-]*)\s+" + r"([a-z][a-z-]*)\s+(.+)$", + lower, + ) + if match: + quality, entity_type, predicate, object_value = match.groups() + before = ( + GraphAtom("$x", "is_a", entity_type), + GraphAtom("$x", "property", quality), + ) + after = (GraphAtom("$x", self.lemma(predicate), _strip_article(object_value)),) + return GraphRule(self._rule_id(before, after), before, after) + return None + + def analyze( + self, + text: str, + discourse: GraphDiscourse, + *, + known_frames: Sequence[Mapping[str, Any]] = (), + ) -> UtteranceGraph: + value = clean(text) + values = _words(value) + lower = value.casefold() + topology: list[str] = [] + if not values: + return UtteranceGraph("delegate") + if values[0] in self.GREETINGS or "good evening" in lower or "good morning" in lower: + return UtteranceGraph( + "social_acknowledgement", payload={"tracks_definitions": "definition" in values}, + topology=("greeting", "capability_question"), confidence=1.0, consumes_as_graph=True, + ) + if set(values) & self.MUTATIONS and set(values) & self.TOOL_NOUNS: + return UtteranceGraph( + "refuse_tool_mutation", topology=("imperative", "mutation", "registered_tool"), + confidence=1.0, consumes_as_graph=True, + ) + tool_domain = bool( + set(values) & self.TOOL_NOUNS + or ("population" in values and set(values) & {"registered", "observations", "table"}) + ) + tool_act = bool( + set(values) & {"consult", "count", "greatest", "highest", "largest", "lookup", "maximum", "minimum", "retrieve", "smallest"} + or "registered" in values + ) + if tool_domain and tool_act: + return UtteranceGraph( + "bounded_tool_request", payload={"surface": value}, + topology=("registered_dataset", "read_only_operation"), confidence=0.98, + consumes_as_graph=True, + ) + if re.match(r"^(?:give|define)\s+(?:the\s+)?(?:established\s+)?(?:technical\s+)?definition\s+of\b", lower): + return UtteranceGraph( + "query_established_definition", payload={"surface": value}, + topology=("definition_imperative", "epistemic_request"), confidence=0.98, + consumes_as_graph=True, + ) + definition = self._definition(value) + if definition is not None: + return UtteranceGraph( + "define_concept", payload={"definition": definition}, + topology=("metalinguistic_connector", "definition_body"), confidence=0.98, + consumes_as_graph=True, + ) + member = self._membership(value, discourse) + if member is not None: + return UtteranceGraph( + "instantiate_concept", payload={"entity": member[0], "concept": member[1]}, + topology=("membership_construction",), confidence=0.98, consumes_as_graph=True, + ) + instance = self._instance_query(value, discourse) + if instance is not None and (value.endswith("?") or any(cue in lower for cue in ("enumerate", "commits", "what can"))): + return UtteranceGraph( + "query_instance", payload={"entity": instance}, topology=("instance_reference", "wh_query"), + confidence=0.94, consumes_as_graph=True, + ) + rule = self._universal_rule(value) + if rule is not None: + return UtteranceGraph( + "assert_rule", payload={"rule": rule}, topology=("universal_quantifier", "condition", "consequence"), + confidence=0.96, consumes_as_graph=True, + ) + numbers = self._number_spans(values) + if len(numbers) >= 2 and ( + set(values) & {"add", "double", "excess", "larger", "minus", "plus", "remove", "subtract", "times"} + ): + operation = "comparison" if {"larger", "excess"} & set(values) else "arithmetic" + return UtteranceGraph( + operation, payload={"numbers": list(numbers), "words": list(values)}, + topology=("numeric_operands", "ordered_operator_sequence"), confidence=0.99, + consumes_as_graph=True, + ) + causal = self._causal(value) + if causal is not None: + return UtteranceGraph( + "assert_causal", atoms=causal, topology=("effect", "because", "cause"), + confidence=0.96, consumes_as_graph=True, + ) + if re.match(r"^what\s+ultimately\s+explains\s+why\b", lower): + return UtteranceGraph( + "query_causal_chain", payload={"surface": value}, topology=("why_query", "transitive_depth"), + confidence=0.96, consumes_as_graph=True, + ) + if lower.startswith("correction"): + return UtteranceGraph( + "correct_state", payload={"surface": value}, topology=("repair_marker", "replacement", "negated_prior"), + confidence=0.96, consumes_as_graph=True, + ) + if lower.startswith(("for a counterfactual", "for the counterfactual")): + return UtteranceGraph( + "assert_counterfactual", payload={"surface": value}, topology=("branch_marker", "replacement"), + confidence=0.96, consumes_as_graph=True, + ) + if "counterfactual" in lower and value.endswith("?"): + return UtteranceGraph( + "query_counterfactual", topology=("branch_query",), confidence=0.96, consumes_as_graph=True, + ) + if lower.startswith("and in actuality") or "actual world" in lower: + return UtteranceGraph( + "query_actual", topology=("parent_world_query",), confidence=0.96, consumes_as_graph=True, + ) + if lower.startswith("imagine ") and " where " in lower and " unless " in lower: + return UtteranceGraph( + "assert_hypothetical", payload={"surface": value}, topology=("world_branch", "rule", "exception"), + confidence=0.96, consumes_as_graph=True, + ) + if "imagined" in lower and "what happens" in lower: + return UtteranceGraph( + "query_hypothetical", payload={"surface": value}, topology=("branch_query", "rule_activation"), + confidence=0.96, consumes_as_graph=True, + ) + if lower.startswith("what blocks"): + return UtteranceGraph( + "query_hypothetical_exception", topology=("exception_query",), confidence=0.96, + consumes_as_graph=True, + ) + if "merely possible" in lower and "required" in lower: + return UtteranceGraph( + "query_modal_strength", payload={"surface": value}, topology=("modal_contrast",), + confidence=0.96, consumes_as_graph=True, + ) + if lower.startswith("returning to") and "where did" in lower: + return UtteranceGraph( + "query_prior_event", topology=("topic_return", "event_role_query"), confidence=0.96, + consumes_as_graph=True, + ) + if re.match(r"^where\s+is\s+.+?\s+relative\s+to\b", lower): + return UtteranceGraph( + "query_spatial", payload={"surface": value}, topology=("relative_location_query",), + confidence=0.96, consumes_as_graph=True, + ) + if re.match(r"^where\s+(?:should\s+i\s+now\s+understand|is|are)\b", lower): + entity_match = re.search( + r"(?:understand\s+|where\s+(?:is|are)\s+)([A-Za-z][A-Za-z0-9_-]*)", + value, re.IGNORECASE, + ) + entity = entity_match.group(1).casefold() if entity_match else None + return UtteranceGraph( + "query_location", payload={"entity": entity}, topology=("location_query",), confidence=0.92, + consumes_as_graph=True, + ) + if " and " in lower and re.search(r"\b(?:north|south|above|below)\s+of\b", lower): + return UtteranceGraph( + "assert_spatial_chain", payload={"surface": value}, topology=("coordinated_relations", "transitive_relation"), + confidence=0.96, consumes_as_graph=True, + ) + if re.match(r"^what\s+follows\s+about\b", lower): + possessive = re.search(r"about\s+([A-Za-z][A-Za-z0-9_-]*)'s\s+(.+?)[?!.]*$", value, re.IGNORECASE) + return UtteranceGraph( + "query_graph_fact", + payload={ + "mode": "consequence", + "subject": possessive.group(1) if possessive else discourse.last_entity, + "anchor": possessive.group(2) if possessive else None, + }, + topology=("consequence_query", "discourse_subject"), confidence=0.96, + consumes_as_graph=True, + ) + if values[0] == "who" and len(values) >= 3: + predicate = self.lemma(values[1]) + graph_predicates = { + fact.atom.predicate for fact in discourse.facts if fact.active + } | { + atom.predicate for rule in discourse.rules for atom in rule.consequents + } + if predicate in graph_predicates: + return UtteranceGraph( + "query_graph_fact", + payload={"mode": "subject", "predicate": predicate, "object": _surface(values[2:])}, + topology=("wh_subject", "graph_predicate", "object_anchor"), confidence=0.96, + consumes_as_graph=True, + ) + frame_query = self._frame_query(value, discourse, known_frames) + if frame_query is not None: + return UtteranceGraph( + "query_frame_role", payload=frame_query, topology=("wh_role", "auxiliary", "known_predicate"), + confidence=0.94, consumes_as_graph=True, + ) + frame = self._initial_adjunct_frame(value) + if frame is not None: + return UtteranceGraph( + "assert_frame", payload={"frame": frame}, topology=("fronted_adjunct", "finite_clause"), + confidence=frame.confidence, consumes_as_graph=True, + ) + if re.match(r"^(?:the\s+)?[A-Za-z][A-Za-z0-9_-]*(?:\s+[A-Za-z][A-Za-z0-9_-]*)*\s+must\s+", value): + # Existing K53 modal algebra is the correct consumer. + return UtteranceGraph("delegate", topology=("modal_assertion",), confidence=0.95) + if "," in value: + trailing = clean(value.split(",", 1)[1]).casefold() + trailing_first = _words(trailing)[:1] + if trailing_first and trailing_first[0] in self.WH | self.IMPERATIVE_CUES: + return UtteranceGraph( + "knowledge_request", topology=("fronted_subordinate", "speech_act_boundary"), + confidence=0.94, + ) + if values[0] in self.IMPERATIVE_CUES or values[0] in self.WH or values[0] in AUXILIARIES: + return UtteranceGraph("knowledge_request", topology=("speech_act_boundary",), confidence=0.9) + + # Declarative states/events enter the discourse graph. This path is + # intentionally last so questions and imperatives cannot be consumed + # merely because they contain a verb. + if not value.endswith("?"): + atom = self._simple_clause(value) + if atom is not None: + return UtteranceGraph( + "assert_fact", atoms=(atom,), topology=("declarative", "predicate_argument"), + confidence=0.78, consumes_as_graph=True, + ) + return UtteranceGraph("delegate", topology=tuple(topology), confidence=0.0) + + +class RelationalGraphTransductionAgent(ConstructionEquivalenceAgent): + """K73 successor with a persistent typed discourse graph in the HRR workspace.""" + + DIRECT_ACTS = frozenset({ + "arithmetic", "assert_causal", "assert_counterfactual", "assert_fact", + "assert_frame", "assert_hypothetical", "assert_rule", "assert_spatial_chain", + "bounded_tool_request", "comparison", "correct_state", "define_concept", "instantiate_concept", + "query_actual", "query_causal_chain", "query_counterfactual", "query_frame_role", + "query_graph_fact", + "query_hypothetical", "query_hypothetical_exception", "query_instance", + "query_location", "query_modal_strength", "query_prior_event", "query_spatial", + "query_established_definition", "refuse_tool_mutation", "social_acknowledgement", + }) + # Small relation families for query-slot projection. These are not facts + # or answers: they connect a requested semantic role to possible corpus + # surfaces. The row is selected only when several independent groups + # converge on the same indexed proposition. + RELATIONAL_BRIDGES: Mapping[str, tuple[str, ...]] = { + "authority": ("power", "government", "administration", "control"), + "denied": ("cannot", "not", "refused", "rejected"), + "denies": ("cannot", "not", "refuses", "rejects"), + "deny": ("cannot", "not", "refuse", "reject"), + "gap": ("distance", "interval", "space"), + "horizon": ("term", "period", "duration", "short", "medium", "long"), + "horizons": ("term", "period", "duration", "short", "medium", "long"), + "institution": ("organization", "agency", "bank", "banks", "government", "court", "school"), + "institutions": ("organizations", "agencies", "bank", "banks", "governments", "courts", "schools"), + "monetary": ("money", "bank", "banks", "central"), + "placement": ("place", "location", "located", "lodged", "ordering", "rests"), + "political": ("government", "governmental", "civic"), + "receiving": ("target", "recipient"), + "restrain": ("control", "limit", "curb", "constrain", "reduce"), + "restrains": ("controls", "limits", "curbs", "constrains", "reduces"), + "synaptic": ("synapse", "synapses"), + } + + @staticmethod + def _cross_segments(text: str) -> tuple[str, str] | None: + """Expose two independently addressable knowledge nuclei. + + The conjunction is a routing boundary, not merely punctuation. Each + side is retrieved under its own specificity certificate before the + propositions are fused in the terminal response. + """ + value = clean(text).rstrip(".?!") + for pattern in ( + r"^juxtapose\s+(.+?)\s+with\s+(.+?)(?:,\s*preserving\b.*)?$", + r"^relate\s+(.+?)\s+to\s+(?:its\s+separate\s+claim\s+about\s+)?(.+?)(?:;\s*do\s+not\b.*)?$", + r"^contrast\s+(.+?)\s+with\s+(.+)$", + r"^state,?\s+(?:without\s+.+?,\s+)?what\s+(.+?)\s+and\s+what\s+(.+)$", + ): + match = re.match(pattern, value, re.IGNORECASE) + if match: + return clean(match.group(1)), clean(match.group(2)) + return None + + def _cross_segment_candidate( + self, segment: str, + ) -> tuple[dict[str, Any] | None, dict[str, Any]]: + candidate, audit = super()._cross_segment_candidate(segment) + if candidate is not None and int(candidate.get("relational_exact_anchor_count", 0)) >= 4: + return candidate, audit + legacy_candidate = candidate + + source_terms = self._relational_source_terms(segment) + links, _ = self.knowledge_basis._semantic_term_links(source_terms) + + # Vote over indexed rows by independent query-role groups. A bridge + # term cannot admit a row alone; at least three situated groups must + # converge. This converts question roles such as INSTITUTION, + # CONTROL, and TIME-HORIZON into an address without treating any one + # WordNet sense as a sufficient answer. + row_groups: dict[str, dict[str, tuple[float, str, str]]] = {} + for term in source_terms: + variants: dict[str, tuple[float, str]] = {} + if term in self.knowledge_basis._term_postings: + variants[term] = (1.0, "literal") + for target, (score, _atom, kind) in links.get(term, {}).items(): + if ( + target in self.knowledge_basis._term_postings + and kind in {"morphological_lexeme", "derivational_family"} + and float(score) >= 0.75 + ): + prior = variants.get(target, (0.0, kind)) + if float(score) > prior[0]: + variants[target] = (float(score), kind) + for target in self.RELATIONAL_BRIDGES.get(term, ()): + if target in self.knowledge_basis._term_postings: + variants[target] = max(variants.get(target, (0.0, "relational_bridge")), (0.90, "relational_bridge")) + for target, (weight, kind) in variants.items(): + information = self.knowledge_basis._information(target) + for text_sha in self.knowledge_basis._term_postings.get(target, ()): + groups = row_groups.setdefault(text_sha, {}) + value = (weight * min(information, 12.0), target, kind) + if value[0] > groups.get(term, (0.0, "", ""))[0]: + groups[term] = value + voted: list[tuple[int, float, int, str, dict[str, tuple[float, str, str]]]] = [] + for text_sha, groups in row_groups.items(): + if len(groups) < 3: + continue + literal_groups = sum(value[2] in {"literal", "morphological_lexeme"} for value in groups.values()) + voted.append((len(groups), sum(value[0] for value in groups.values()), literal_groups, text_sha, groups)) + voted.sort(key=lambda value: (-value[0], -value[1], -value[2], value[3])) + if voted: + group_count, vote, literal_groups, text_sha, groups = voted[0] + raw = dict(self.knowledge_basis._rows[text_sha]) + primary = str(raw.get("required_domain_ids", [raw.get("domain_id", "")])[0]) + row = { + "domain_id": primary, + "text": raw.get("text", ""), + "source_document": raw.get("source_document", "unknown"), + "source_document_sha256": raw.get("source_document_sha256", ""), + "text_sha256": raw.get("text_sha256", text_sha), + "license_note": raw.get("license_note", ""), + "source_split": raw.get("split", ""), + "retrieval_score": vote / max(1, len(source_terms)), + "answer_status": "relational_role_evidence_only", + "retrieval_path": "k76_independent_role_group_convergence", + } + return row, { + **audit, + "relation_kind": "k76_independent_role_group_convergence", + "relational_group_count": group_count, + "literal_group_count": literal_groups, + "relational_vote": vote, + "group_surfaces": { + key: {"target": value[1], "kind": value[2]} + for key, value in groups.items() + }, + "admitted": True, + } + + # Long question scaffolds can lower an otherwise decisive exact-row + # margin. Re-evaluate bounded prefixes of the rarest relational + # nucleus. Two anchors are accepted only when their joint information + # and separation from the runner-up are strong; one-anchor guesses are + # never admitted. + forms: list[tuple[str, ...]] = [] + literal = tuple(term for term in source_terms if term in self.knowledge_basis._term_postings) + if literal: + forms.append(literal) + projected: list[str] = [] + for term in source_terms: + alternatives = [ + (target, score, kind) + for target, (score, _atom, kind) in links.get(term, {}).items() + if target in self.knowledge_basis._term_postings + and kind in {"morphological_lexeme", "derivational_family"} + and float(score) >= 0.75 + ] + alternatives.sort(key=lambda value: ( + -self.knowledge_basis._information(value[0]), -float(value[1]), value[0], + )) + projected.append(alternatives[0][0] if alternatives else term) + if projected: + forms.append(tuple(projected)) + admitted: list[tuple[int, float, float, dict[str, Any], str]] = [] + for form in forms: + unique = tuple(dict.fromkeys(term for term in form if term in self.knowledge_basis._term_postings)) + ranked = sorted( + unique, + key=lambda term: (-self.knowledge_basis._information(term), term), + )[:8] + for size in range(2, len(ranked) + 1): + query = " ".join(ranked[:size]) + values = self.knowledge_basis._specific_exact_candidates(query, None) + if not values: + continue + information, _lexical, row = values[0] + anchors = int(row.get("relational_exact_anchor_count", 0)) + margin = float(row.get("relational_exact_information_margin", 0.0)) + if anchors >= 2 and float(information) >= 14.0 and margin >= 2.0: + admitted.append((anchors, float(information), margin, dict(row), query)) + if admitted: + admitted.sort(key=lambda value: (-value[0], -value[1], -value[2], value[3]["text_sha256"])) + anchors, information, margin, row, query = admitted[0] + return row, { + **audit, + "query_form": query, + "relation_kind": "k76_information_separated_relational_nucleus", + "exact_anchor_count": anchors, + "exact_information": information, + "exact_information_margin": margin, + "admitted": True, + } + # Finally use the frozen concept-address lattice as a convergent graph. + # It remains useful for terse two-anchor segments, but comes after the + # independent role-group certificate because polysemous gloss matches + # alone are not a reliable basis for open-domain source selection. + semantic = self.knowledge_basis._semantic_candidates( + segment, self.knowledge_basis.corpus_path, None, + ) + if semantic: + score, row = semantic[0] + return dict(row), { + **audit, + "relation_kind": "k76_convergent_semantic_nucleus", + "semantic_score": float(score), + "admitted": True, + } + if legacy_candidate is not None: + return legacy_candidate, audit + return None, {**audit, "k76_relational_nucleus_admitted": False} + + @staticmethod + def _knowledge_speech_act(text: str) -> bool: + if ConstructionEquivalenceAgent._knowledge_speech_act(text): + return True + value = clean(text).casefold() + if "," in value: + trailing = clean(value.split(",", 1)[1]) + return bool(re.match( + r"^(?:describe|explain|identify|state|report|compare|relate|contrast|" + r"what|which|who|how|where|when|why)\b", + trailing, + )) + return False + + def __init__(self, *args: Any, **kwargs: Any) -> None: + kwargs.setdefault("agent_id", "oracle-stage5k76-relational-graph-transduction-agent") + super().__init__(*args, **kwargs) + self.graph_transducer = ClosedClassGraphTransducer(self) + self._k76_discourse: dict[str, GraphDiscourse] = {} + self._k76_locks: dict[str, threading.RLock] = {} + self._k76_current: ContextVar[UtteranceGraph | None] = ContextVar( + "oracle_k76_current_graph", default=None, + ) + lineage = hashlib.sha256() + for name in ("stage5k73_construction_equivalence_agent.py", Path(__file__).name): + payload = Path(__file__).with_name(name).read_bytes() + lineage.update(len(payload).to_bytes(8, "big")); lineage.update(payload) + self._k76_lineage_sha256 = lineage.hexdigest() + + def _graph_discourse(self, session_id: str) -> GraphDiscourse: + if session_id not in self._k76_discourse: + self._k76_discourse[session_id] = GraphDiscourse() + self._k76_locks[session_id] = threading.RLock() + return self._k76_discourse[session_id] + + @staticmethod + def _graph_from_context(context: Mapping[str, Any] | None) -> Mapping[str, Any]: + value = dict(context or {}).get("k76_semantic_graph", {}) + return value if isinstance(value, Mapping) else {} + + @classmethod + def _intent_lattice( + cls, text: str, context: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + audit = dict(super()._intent_lattice(text, context)) + graph = cls._graph_from_context(context) + act = str(graph.get("speech_act", "delegate")) + if act in cls.DIRECT_ACTS: + scores = dict(audit.get("candidate_scores", {})) + scores["ephemeral_graph"] = 1.0 + audit["candidate_scores"] = scores + audit["selected_intent_family"] = "ephemeral_graph" + audit["selection_policy"] = "k76_whole_utterance_graph_before_local_predicate" + audit["k76_graph_dispatch"] = { + "speech_act": act, + "whole_utterance_precedes_predicate": True, + "answer_or_gold_labels_used": False, + } + elif act == "knowledge_request": + audit["selected_intent_family"] = "open" + audit["selection_policy"] = "k76_speech_act_boundary_before_event_induction" + return audit + + @classmethod + def _routing_goal( + cls, text: str, context: Mapping[str, Any], + ) -> tuple[tuple[str, ...], ...]: + act = str(cls._graph_from_context(context).get("speech_act", "delegate")) + if act in {"arithmetic", "comparison"}: + return (("numeric", "tool_router"), ("linguistic",)) + if act in {"define_concept", "instantiate_concept", "query_instance"}: + return (("definition", "linguistic"), ("state",)) + if act in {"assert_rule", "assert_causal", "query_causal_chain", "assert_hypothetical", "query_hypothetical", "query_hypothetical_exception"}: + return (("causal", "mechanism"), ("state", "event"), ("linguistic",)) + if act in {"refuse_tool_mutation"}: + return (("tool_router", "epistemic"), ("linguistic",)) + if act == "bounded_tool_request": + return (("tool_router", "dataset"), ("linguistic",)) + if act == "query_established_definition": + return (("definition", "epistemic"), ("linguistic",)) + if act in cls.DIRECT_ACTS: + return (("event", "state", "spatial"), ("linguistic",)) + return super()._routing_goal(text, context) + + @staticmethod + def _atom_key(atom: GraphAtom) -> str: + return digest(atom.as_dict()) + + @staticmethod + def _same_surface(left: str, right: str) -> bool: + one = set(_words(_strip_article(left))) + two = set(_words(_strip_article(right))) + return bool(one and two and (one <= two or two <= one)) + + def _store_atom( + self, + session_id: str, + atom: GraphAtom, + *, + inferred: bool = False, + supports: Sequence[str] = (), + ) -> StoredGraphFact: + discourse = self._graph_discourse(session_id) + duplicate = next(( + value for value in reversed(discourse.facts) + if value.active and value.atom == atom + ), None) + if duplicate is not None: + return duplicate + session = self._session(session_id) + relation = atom.predicate if atom.polarity == "positive" else "not::" + atom.predicate + state = self._relation_state( + session, atom.subject, relation, atom.object_value, + world_id=None if atom.world == "actual" else atom.world, + ) + fact = StoredGraphFact(atom, state.state_id, True, inferred, tuple(supports)) + discourse.facts.append(fact) + session.facts.append(RelationFact( + atom.subject, relation, atom.object_value, + state.state_id, state.world_id, inferred=inferred, + )) + discourse.last_entity = atom.subject + if atom.predicate not in {"is_a", "property"}: + discourse.last_event_subject = atom.subject + discourse.last_event_object = atom.object_value + destination = next((value for role, value in atom.qualifiers if role in {"to", "into", "through"}), None) + if destination: + discourse.last_event_destination = destination + if atom.predicate == "property": + discourse.last_property_entity = atom.subject + return fact + + @staticmethod + def _unify(pattern: GraphAtom, fact: GraphAtom, bindings: Mapping[str, str]) -> dict[str, str] | None: + if pattern.predicate != fact.predicate or pattern.world != fact.world: + return None + result = dict(bindings) + for expected, observed in ((pattern.subject, fact.subject), (pattern.object_value, fact.object_value)): + if expected.startswith("$"): + bound = result.get(expected) + if bound is not None and bound.casefold() != observed.casefold(): + return None + result[expected] = observed + elif expected.casefold() != observed.casefold(): + return None + return result + + @staticmethod + def _instantiate(atom: GraphAtom, bindings: Mapping[str, str]) -> GraphAtom: + return GraphAtom( + bindings.get(atom.subject, atom.subject), atom.predicate, + bindings.get(atom.object_value, atom.object_value), atom.world, + atom.polarity, atom.qualifiers, + ) + + def _rule_closure(self, session_id: str) -> tuple[StoredGraphFact, ...]: + discourse = self._graph_discourse(session_id) + created: list[StoredGraphFact] = [] + for _ in range(max(1, len(discourse.rules) + 1)): + changed = False + active = [value for value in discourse.facts if value.active] + for rule in discourse.rules: + bindings: list[tuple[dict[str, str], tuple[str, ...]]] = [({}, ())] + for antecedent in rule.antecedents: + next_bindings: list[tuple[dict[str, str], tuple[str, ...]]] = [] + for current, supports in bindings: + for fact in active: + unified = self._unify(antecedent, fact.atom, current) + if unified is not None: + next_bindings.append((unified, (*supports, fact.state_id))) + bindings = next_bindings + if not bindings: + break + for current, supports in bindings: + if rule.exception is not None: + exception = self._instantiate(rule.exception, current) + if any(value.active and value.atom == exception for value in active): + continue + for consequent in rule.consequents: + atom = self._instantiate(consequent, current) + if any(value.active and value.atom == atom for value in discourse.facts): + continue + created.append(self._store_atom( + session_id, atom, inferred=True, supports=supports, + )) + active = [value for value in discourse.facts if value.active] + changed = True + if not changed: + break + return tuple(created) + + @staticmethod + def _atom_clause(atom: GraphAtom) -> SemanticClause: + if atom.predicate == "is_a": + return SemanticClause(atom.subject, "be", complement=f"a {atom.object_value}") + if atom.predicate == "property": + return SemanticClause(atom.subject, "be", complement=atom.object_value) + if atom.predicate in {"inside", "within", "beneath", "above", "below", "north", "south"}: + connector = "of" if atom.predicate in {"north", "south"} else "" + return SemanticClause( + atom.subject, "be", + complement=clean(f"{atom.predicate} {connector} {atom.object_value}"), + ) + return SemanticClause( + atom.subject, atom.predicate, + object_value=None if atom.object_value == "event" else atom.object_value, + qualifiers=atom.qualifiers, + tense="present", + polarity=atom.polarity, + ) + + @staticmethod + def _past_atom_clause(atom: GraphAtom) -> SemanticClause: + clause = RelationalGraphTransductionAgent._atom_clause(atom) + if atom.predicate in {"is_a", "property", "be"}: + return clause + return SemanticClause( + clause.subject, clause.predicate, clause.object_value, clause.complement, + clause.qualifiers, tense="past", modal=clause.modal, polarity=clause.polarity, + ) + + def _record_graph_operation( + self, session_id: str, operation: str, facts: Sequence[StoredGraphFact], **details: Any, + ) -> None: + session = self._session(session_id) + states = [] + by_id = {value.state.state_id: value.state for value in session.workspace.records} + for fact in facts: + if fact.state_id in by_id: + states.append(by_id[fact.state_id]) + self._record_last( + session, operation, tuple(states), + typed_graph=True, graph_fact_count=len(facts), **details, + ) + + def _define(self, session_id: str, definition: GraphDefinition) -> HandlerResult: + session = self._session(session_id) + variable = session.workspace.new_variable("x", "entity") + x = RelationalValue.variable(variable) + states = [] + states.append(session.workspace.compose( + "relate", (x, self._relation(session, "is_a"), self._entity(session, definition.kind)), + ttl_turns=128, + )) + for quality in definition.properties: + states.append(session.workspace.compose( + "property", (x, session.workspace.symbol(quality, "quality")), ttl_turns=128, + )) + if definition.shape: + states.append(session.workspace.compose( + "shape", (x, session.workspace.symbol(definition.shape, "shape")), ttl_turns=128, + )) + if definition.relation and definition.relation_object: + states.append(session.workspace.compose( + "relate", ( + x, self._relation(session, definition.relation), + self._entity(session, definition.relation_object), + ), ttl_turns=128, + )) + body = states[0] + for state in states[1:]: + body = session.workspace.compose("and", (body, state), ttl_turns=128) + states.append(body) + bound = session.workspace.define_concept(definition.term, variable, body, ttl_turns=128) + stored = replace(definition, definition_state_id=bound.definition_state_id) + discourse = self._graph_discourse(session_id) + discourse.definitions[definition.term] = stored + discourse.last_entity = definition.term + session.concepts.add(definition.term) + clauses = [SemanticClause(f"A {definition.term}", "be", complement=quality) for quality in definition.properties] + if definition.shape: + clauses.append(SemanticClause(f"A {definition.term}", "be", complement=f"{definition.shape}-shaped")) + if definition.relation and definition.relation_object: + clauses.append(SemanticClause(f"A {definition.term}", definition.relation, object_value=definition.relation_object)) + self._record_last( + session, "define_ephemeral_concept", tuple(states), concept=definition.term, + definition_state_id=bound.definition_state_id, variable_bound=True, + semantic_graph=stored.as_dict(), + ) + return HandlerResult( + "define_ephemeral_concept", "answered", (("concept", definition.term),), + (("definition_state_id", bound.definition_state_id),), tuple(clauses), + (("source", "conversation_local_graph_definition"),), + ("induce_definition_graph", "bind_typed_variable", "compose_definition_body", "stabilize_tree_route"), + ) + + def _instantiate_concept(self, session_id: str, entity: str, concept: str) -> HandlerResult: + session = self._session(session_id) + discourse = self._graph_discourse(session_id) + definition = discourse.definitions[concept] + state = session.workspace.instantiate_concept( + concept, self._entity(session, entity), ttl_turns=128, + ) + session.instances[entity.casefold()] = state + discourse.instances[entity.casefold()] = concept + discourse.last_entity = entity.casefold() + self._record_last( + session, "instantiate_ephemeral_concept", (state,), entity=entity, + concept=concept, definition_state_id=definition.definition_state_id, + graph_substitution=True, + ) + return HandlerResult( + "instantiate_ephemeral_concept", "answered", + (("entity", entity), ("concept", concept)), (("instance_state_id", state.state_id),), + (SemanticClause(entity, "be", complement=f"a {concept}"),), + (("source", "conversation_local_graph_definition"),), + ("retrieve_graph_definition", "substitute_bound_entity", "stabilize_instantiated_graph"), + ) + + def _query_instance(self, session_id: str, entity: str) -> HandlerResult: + discourse = self._graph_discourse(session_id) + concept = discourse.instances.get(entity.casefold()) + if concept is None: + return self._abstention("query_ephemeral_instance", entity, "instance_not_in_graph_discourse") + definition = discourse.definitions[concept] + clauses = [SemanticClause(entity, "be", complement=quality) for quality in definition.properties] + if definition.shape: + clauses.append(SemanticClause(entity, "be", complement=f"{definition.shape}-shaped")) + if definition.relation and definition.relation_object: + clauses.append(SemanticClause(entity, definition.relation, object_value=definition.relation_object)) + self._record_last( + self._session(session_id), "query_ephemeral_instance", (), entity=entity, + concept=concept, clause_count=len(clauses), graph_definition_inverse=True, + ) + return HandlerResult( + "query_ephemeral_instance", "answered", (("entity", entity),), + (("derived_clauses", len(clauses)),), tuple(clauses), + (("source", "ephemeral_graph_definition"),), + ("resolve_instance_reference", "retrieve_definition_graph", "realize_atomic_relations"), + ) + + def _assert_fact(self, session_id: str, atom: GraphAtom) -> HandlerResult: + facts: list[StoredGraphFact] = [] + # Copular type-with-location constructions bind two relations rather + # than treating the prepositional tail as part of the type label. + if atom.predicate == "is_a" and atom.qualifiers: + facts.append(self._store_atom( + session_id, + GraphAtom(atom.subject, "is_a", atom.object_value, atom.world), + )) + for relation, object_value in atom.qualifiers: + facts.append(self._store_atom( + session_id, + GraphAtom(atom.subject, relation, object_value, atom.world), + )) + elif atom.predicate == "property" and re.search(r"\s+(?:inside|within)\s+", atom.object_value): + match = re.match(r"(?:a|an)?\s*([a-z][a-z-]*)\s+(inside|within)\s+(?:a|an|the)?\s*(.+)", atom.object_value) + if match: + facts.append(self._store_atom(session_id, GraphAtom(atom.subject, "is_a", match.group(1)))) + facts.append(self._store_atom(session_id, GraphAtom(atom.subject, match.group(2), _strip_article(match.group(3))))) + if not facts: + facts.append(self._store_atom(session_id, atom)) + inferred = self._rule_closure(session_id) + facts.extend(inferred) + self._record_graph_operation(session_id, "assert_graph_fact", facts, closure_count=len(inferred)) + return HandlerResult( + "assert_graph_fact", "answered", + (("facts", len(facts)),), (("inferred", len(inferred)),), + tuple(self._atom_clause(value.atom) for value in facts), + (("source", "conversation_local_graph"),), + ("bind_typed_graph_atom", "run_variable_unification", "compute_rule_closure", "release_graph_inverse"), + ) + + def _assert_rule(self, session_id: str, rule: GraphRule) -> HandlerResult: + discourse = self._graph_discourse(session_id) + if not any(value.rule_id == rule.rule_id for value in discourse.rules): + discourse.rules.append(rule) + session = self._session(session_id) + states = [] + for atom in (*rule.antecedents, *rule.consequents): + states.append(self._relation_state(session, atom.subject, atom.predicate, atom.object_value)) + antecedent = states[0] + for state in states[1:len(rule.antecedents)]: + antecedent = session.workspace.compose("and", (antecedent, state), ttl_turns=128) + states.append(antecedent) + consequence = states[len(rule.antecedents)] + condition = session.workspace.compose("condition", (antecedent, consequence), ttl_turns=128) + states.append(condition) + self._record_last( + session, "compose_graph_rule", tuple(states), rule=rule.as_dict(), + variable_scope_preserved=True, + ) + def rule_clause(atom: GraphAtom) -> SemanticClause: + if atom.predicate in {"is_a", "property", "inside", "within", "beneath", "above", "below"}: + return self._atom_clause(atom) + return SemanticClause( + atom.subject, atom.predicate, + object_value=None if atom.object_value == "event" else atom.object_value, + ) + + clauses_list: list[SemanticClause] = [] + consumed: set[int] = set() + for index, atom in enumerate(rule.antecedents): + if index in consumed or atom.predicate != "is_a": + continue + property_index = next(( + offset for offset, other in enumerate(rule.antecedents) + if offset not in consumed + and other.subject == atom.subject + and other.predicate == "property" + ), None) + if property_index is not None: + quality = rule.antecedents[property_index].object_value + clauses_list.append(SemanticClause( + atom.subject, "be", complement=f"a {quality} {atom.object_value}", + )) + consumed.update((index, property_index)) + clauses_list.extend( + rule_clause(atom) + for index, atom in enumerate(rule.antecedents) + if index not in consumed + ) + clauses_list.extend(rule_clause(atom) for atom in rule.consequents) + clauses = tuple(clauses_list) + return HandlerResult( + "compose_graph_rule", "answered", (("rule_id", rule.rule_id),), + (("antecedents", len(rule.antecedents)), ("consequents", len(rule.consequents))), + clauses, (("source", "conversation_local_graph_rule"),), + ("bind_rule_variables", "compose_antecedent_graph", "compose_consequence_graph", "retain_scope"), + ) + + def _query_frame_role(self, session_id: str, payload: Mapping[str, Any]) -> HandlerResult: + session = self._session(session_id) + predicate = str(payload["predicate"]) + target = str(payload["target_role"]) + anchors = tuple(str(value) for value in payload.get("anchors", ())) + for frame in self._stored_frames(session): + if str(frame.get("lemma")) != predicate: + continue + roles = {str(key): str(value) for key, value in frame.get("roles", {}).items()} + if target not in roles: + continue + surfaces = tuple(roles.values()) + relevant = [anchor for anchor in anchors if anchor not in {predicate, _singular(predicate)}] + if relevant and not all(any(self._same_surface(anchor, surface) for surface in surfaces) for anchor in relevant[-2:]): + # Anchors are a preference, not a hard parse commitment; a + # unique predicate frame still determines the role safely. + same_predicate = [value for value in self._stored_frames(session) if str(value.get("lemma")) == predicate] + if len(same_predicate) != 1: + continue + value = roles[target] + self._record_last( + session, "query_induced_role", (), queried_predicate=predicate, + queried_role=target, released_value=value, induced_frame=frame, + graph_role_resolution=True, + ) + return HandlerResult( + "query_induced_role", "answered", (("predicate", predicate), ("role", target)), + (("role", target), ("value", value)), + (self._clause(frame),), (("source", "conversation_local_induced_frame"),), + ("resolve_wh_role", "match_typed_frame", "release_selected_role", "realize_frame_inverse"), + ) + return self._abstention("query_induced_role", predicate, "congruent_graph_frame_not_in_workspace") + + def _query_graph_fact(self, session_id: str, payload: Mapping[str, Any]) -> HandlerResult: + discourse = self._graph_discourse(session_id) + active = [value for value in discourse.facts if value.active] + mode = str(payload.get("mode", "subject")) + if mode == "subject": + predicate = str(payload.get("predicate", "")) + object_value = str(payload.get("object", "")) + fact = next(( + value for value in reversed(active) + if value.atom.predicate == predicate + and self._same_surface(object_value, value.atom.object_value) + ), None) + if fact is None: + return self._abstention("query_graph_fact", object_value, "matching_graph_fact_absent") + selected = (fact,) + else: + subject = str(payload.get("subject") or "") + anchor = str(payload.get("anchor") or "") + candidates = [ + value for value in active + if self._same_surface(subject, value.atom.subject) + and value.atom.predicate not in {"is_a", "property"} + and ( + not anchor + or self._same_surface(anchor, value.atom.object_value) + or self._same_surface(anchor, value.atom.predicate) + ) + ] + inferred = [value for value in candidates if value.inferred] + selected = tuple(inferred or candidates[-1:]) + if not selected: + return self._abstention("query_graph_fact", subject, "graph_consequence_absent") + self._record_graph_operation( + session_id, "query_graph_fact", selected, + query_mode=mode, released_fact_count=len(selected), + ) + return HandlerResult( + "query_graph_fact", "answered", (("mode", mode),), + (("facts", [value.atom.as_dict() for value in selected]),), + tuple(self._atom_clause(value.atom) for value in selected), + (("source", "conversation_local_graph"),), + ("resolve_query_roles", "select_active_graph_fact", "release_graph_inverse"), + ) + + def _execute_graph_arithmetic(self, graph: UtteranceGraph) -> HandlerResult: + numbers = [int(value) for value in graph.payload.get("numbers", ())] + words = set(str(value) for value in graph.payload.get("words", ())) + if graph.speech_act == "comparison": + high, low = max(numbers[:2]), min(numbers[:2]) + difference = high - low + clauses = ( + SemanticClause("The larger quantity", "be", complement=str(high)), + SemanticClause("The smaller quantity", "be", complement=str(low)), + SemanticClause("The excess", "be", complement=str(difference)), + ) + value: float = float(difference) + display = str(difference) + else: + value = float(numbers[0]) + for operand in numbers[1:]: + if "double" in words: + value *= 2 + words.discard("double") + # The first following numeric is the subtraction operand, + # not a multiplier encoded by the word ``double``. + if operand == numbers[0]: + continue + if words & {"remove", "subtract", "minus"}: + value -= operand + elif words & {"add", "plus"}: + value += operand + elif words & {"times", "multiply"}: + value *= operand + # For ``begin with N, double it, remove M``, number spans contain + # N and M; apply the ordered unary operator exactly once. + if "double" in set(graph.payload.get("words", ())): + value = float(numbers[0] * 2 - numbers[-1]) + display = str(int(value)) if value.is_integer() else str(value) + clauses = (SemanticClause("The result", "be", complement=display),) + return HandlerResult( + "bounded_arithmetic", "answered", (("operand_count", len(numbers)),), + (("value", value), ("display", display)), clauses, + (("tool", "native_graph_arithmetic"),), + ("bind_numeric_operands", "execute_ordered_graph_operators", "verify_numeric_result"), + tool_audit={"tool": "native_graph_arithmetic", "read_only": True, "result": display}, + ) + + def _assert_causal(self, session_id: str, atoms: Sequence[GraphAtom]) -> HandlerResult: + cause, effect = atoms + cause_fact = self._store_atom(session_id, cause) + effect_fact = self._store_atom(session_id, effect) + discourse = self._graph_discourse(session_id) + edge = (cause_fact.state_id, effect_fact.state_id) + if edge not in discourse.causal_edges: + discourse.causal_edges.append(edge) + session = self._session(session_id) + by_id = {value.state.state_id: value.state for value in session.workspace.records} + cause_state, effect_state = by_id[cause_fact.state_id], by_id[effect_fact.state_id] + edge_state = session.workspace.compose("cause", (cause_state, effect_state), ttl_turns=128) + self._record_last( + session, "compose_causal_graph", (cause_state, effect_state, edge_state), + cause=cause.as_dict(), effect=effect.as_dict(), + ) + return HandlerResult( + "compose_causal_graph", "answered", (("cause", cause.as_dict()), ("effect", effect.as_dict())), + (("edge_state_id", edge_state.state_id),), + (self._past_atom_clause(effect), self._past_atom_clause(cause)), + (("source", "conversation_local_causal_graph"),), + ("bind_cause_atom", "bind_effect_atom", "compose_directed_causal_edge"), + ) + + def _query_causal_chain(self, session_id: str) -> HandlerResult: + discourse = self._graph_discourse(session_id) + active = {value.state_id: value for value in discourse.facts if value.active} + if not discourse.causal_edges: + return self._abstention("query_causal_chain", "the requested causal chain", "causal_graph_empty") + effect_id = discourse.causal_edges[-1][1] + # Prefer the effect furthest downstream (an id with incoming but no + # outgoing edge), independent of the order in which premises arrived. + outgoing = {left for left, _ in discourse.causal_edges} + sinks = [right for _, right in discourse.causal_edges if right not in outgoing] + if sinks: + effect_id = sinks[-1] + chain = [effect_id] + while True: + edge = next(((left, right) for left, right in reversed(discourse.causal_edges) if right == chain[-1]), None) + if edge is None or edge[0] in chain: + break + chain.append(edge[0]) + ordered = [active[value] for value in reversed(chain) if value in active] + if len(ordered) < 2: + return self._abstention("query_causal_chain", "the requested causal chain", "causal_path_incomplete") + self._record_graph_operation(session_id, "query_causal_chain", ordered, path_length=len(ordered)) + return HandlerResult( + "query_causal_chain", "answered", (("path_length", len(ordered)),), + (("atoms", [value.atom.as_dict() for value in ordered]),), + tuple(self._past_atom_clause(value.atom) for value in ordered), + (("source", "conversation_local_causal_graph"),), + ("select_downstream_effect", "traverse_reverse_causal_edges", "release_transitive_explanation"), + ) + + def _correct_state(self, session_id: str, surface: str) -> HandlerResult: + discourse = self._graph_discourse(session_id) + session = self._session(session_id) + lower = clean(surface).casefold() + match = re.search(r"\b(?:went|moved|entered)\s+(?:into|to)\s+(?:the\s+)?(.+?)(?:,|\s+not\b)", lower) + destination = _strip_article(match.group(1)) if match else None + subject = discourse.last_event_subject or discourse.last_entity + if not destination or not subject: + return self._abstention("correct_graph_state", surface, "repair_roles_unresolved") + retracted: list[str] = [] + for fact in reversed(discourse.facts): + if fact.active and fact.atom.subject.casefold() == subject.casefold() and fact.atom.predicate in {"enter", "go", "move", "location"}: + fact.active = False + retracted.append(fact.state_id) + if session.workspace.is_visible(fact.state_id): + session.workspace.retract(fact.state_id, reason="discourse_correction") + new_fact = self._store_atom(session_id, GraphAtom(subject, "location", destination)) + self._record_graph_operation( + session_id, "correct_graph_state", (new_fact,), retracted_state_ids=retracted, + supersession=True, + ) + return HandlerResult( + "correct_graph_state", "answered", (("entity", subject),), + (("location", destination), ("retracted", retracted)), + (SemanticClause(subject, "be", complement=f"in the {destination}"),), + (("source", "conversation_local_correction"),), + ("resolve_anaphoric_subject", "retract_superseded_state", "bind_replacement_state"), + ) + + def _query_location(self, session_id: str, entity: str | None) -> HandlerResult: + discourse = self._graph_discourse(session_id) + subject = entity or discourse.last_entity + fact = next(( + value for value in reversed(discourse.facts) + if value.active and value.atom.predicate in {"location", "enter", "go", "move"} + and (subject is None or self._same_surface(subject, value.atom.subject)) + ), None) + if fact is None: + return self._abstention("query_graph_location", subject or "the entity", "active_location_absent") + self._record_graph_operation(session_id, "query_graph_location", (fact,), entity=fact.atom.subject) + return HandlerResult( + "query_graph_location", "answered", (("entity", fact.atom.subject),), + (("location", fact.atom.object_value),), + (SemanticClause(fact.atom.subject, "be", complement=f"in the {fact.atom.object_value}"),), + (("source", "active_conversation_graph"),), + ("resolve_entity_reference", "select_non_superseded_location", "release_location"), + ) + + def _execute_graph_counterfactual(self, session_id: str, surface: str) -> HandlerResult: + discourse = self._graph_discourse(session_id) + match = re.search(r"\breplace\s+(.+?)\s+with\s+(.+?)[.!?]*$", clean(surface), re.IGNORECASE) + entity = discourse.last_property_entity + if match is None or entity is None: + return self._abstention("branch_counterfactual_graph", surface, "counterfactual_roles_unresolved") + old, new = _strip_article(match.group(1)), _strip_article(match.group(2)) + world = self._session(session_id).workspace.branch_world("k76_counterfactual") + atom = GraphAtom(entity, "property", new, world=world) + fact = self._store_atom(session_id, atom) + discourse.current_counterfactual_entity = entity + self._record_graph_operation( + session_id, "branch_counterfactual_graph", (fact,), entity=entity, + replaced=old, replacement=new, parent_unchanged=True, + ) + return HandlerResult( + "branch_counterfactual_graph", "answered", (("entity", entity),), + (("counterfactual_quality", new),), + (SemanticClause(entity, "be", complement=new, qualifiers=(("in", "the counterfactual"),)),), + (("source", "conversation_local_counterfactual_graph"),), + ("branch_world", "bind_local_replacement", "preserve_parent_world"), + ) + + def _query_world_property(self, session_id: str, *, counterfactual: bool) -> HandlerResult: + discourse = self._graph_discourse(session_id) + entity = discourse.current_counterfactual_entity or discourse.last_property_entity + candidates = [ + value for value in discourse.facts + if value.active and value.atom.predicate == "property" + and entity is not None and value.atom.subject.casefold() == entity.casefold() + and ((value.atom.world != "actual") == counterfactual) + ] + if not candidates: + return self._abstention("query_world_property", entity or "the entity", "world_property_absent") + fact = candidates[-1] + world_label = "counterfactual" if counterfactual else "actuality" + self._record_graph_operation(session_id, "query_world_property", (fact,), world=world_label) + return HandlerResult( + "query_world_property", "answered", (("entity", fact.atom.subject), ("world", world_label)), + (("quality", fact.atom.object_value),), + (SemanticClause(fact.atom.subject, "be", complement=fact.atom.object_value, qualifiers=(("in", world_label),)),), + (("source", "branch_selected_graph_state"),), + ("select_requested_world", "verify_visibility", "release_world_local_property"), + ) + + def _execute_graph_hypothetical(self, session_id: str, surface: str) -> HandlerResult: + match = re.match(r"^imagine\s+(.+?)\s+where\s+(.+?)\s+unless\s+(.+?)[.!?]*$", clean(surface), re.IGNORECASE) + if match is None: + return self._abstention("compose_hypothetical_graph", surface, "hypothetical_topology_unresolved") + world_name, main_surface, exception_surface = match.groups() + world = self._session(session_id).workspace.branch_world( + "k76_hypothetical:" + re.sub(r"\W+", "_", world_name.casefold()), + ) + main = self.graph_transducer._simple_clause(main_surface, world=world) + exception = self.graph_transducer._simple_clause(exception_surface, world=world) + if main is None or exception is None: + return self._abstention("compose_hypothetical_graph", surface, "hypothetical_atoms_unresolved") + # Caused-action surface ``a bell opens every door`` is represented as + # a condition on the bell event with the doors' state as consequence. + antecedent = GraphAtom(main.subject, "sound", "event", world=world) + consequence_subject = main.object_value + every = re.match(r"^every\s+([a-z][a-z-]*)$", consequence_subject, re.IGNORECASE) + if every: + noun = every.group(1).casefold() + plural = noun[:-1] + "ies" if noun.endswith("y") else noun + "s" + consequence_subject = "The " + plural + consequence = GraphAtom(consequence_subject, main.predicate, "event", world=world) + rule = GraphRule( + "hypothetical:" + digest([main.as_dict(), exception.as_dict()])[:24], + (antecedent,), (consequence,), world=world, exception=exception, + ) + discourse = self._graph_discourse(session_id) + discourse.current_hypothetical_rule = rule + discourse.rules.append(rule) + session = self._session(session_id) + before_state = self._relation_state(session, antecedent.subject, antecedent.predicate, antecedent.object_value, world_id=world) + after_state = self._relation_state(session, consequence.subject, consequence.predicate, consequence.object_value, world_id=world) + exception_state = self._relation_state(session, exception.subject, exception.predicate, exception.object_value, world_id=world) + exception_absent = session.workspace.compose( + "negate", (exception_state,), world_id=world, ttl_turns=128, + ) + enabled = session.workspace.compose( + "and", (before_state, exception_absent), world_id=world, ttl_turns=128, + ) + guarded = session.workspace.compose( + "condition", (enabled, after_state), world_id=world, ttl_turns=128, + ) + self._record_last( + session, "compose_hypothetical_graph", + (before_state, after_state, exception_state, exception_absent, enabled, guarded), + rule=rule.as_dict(), branch_local=True, + ) + return HandlerResult( + "compose_hypothetical_graph", "answered", (("world", world_name),), + (("rule_id", rule.rule_id),), + ( + SemanticClause(consequence.subject, consequence.predicate, qualifiers=(("when", f"{antecedent.subject} {antecedent.predicate}s"),)), + SemanticClause(exception.subject, "be", complement=exception.object_value), + ), + (("source", "conversation_local_hypothetical_graph"),), + ("branch_world", "bind_trigger", "bind_consequence", "bind_exception_guard"), + ) + + def _query_hypothetical(self, session_id: str, *, exception: bool = False) -> HandlerResult: + discourse = self._graph_discourse(session_id) + rule = discourse.current_hypothetical_rule + if rule is None: + return self._abstention("query_hypothetical_graph", "the imagined result", "hypothetical_graph_absent") + atom = rule.exception if exception else rule.consequents[0] + assert atom is not None + return HandlerResult( + "query_hypothetical_graph", "answered", (("rule_id", rule.rule_id),), + (("selected", "exception" if exception else "consequence"),), + (self._atom_clause(atom),), (("source", "conversation_local_hypothetical_graph"),), + ("select_branch_rule", "select_exception" if exception else "resolve_satisfied_guard", "release_graph_atom"), + ) + + def _query_modal_strength(self, session_id: str) -> HandlerResult: + session = self._session(session_id) + fact = next(( + value for value in reversed(session.facts) + if value.relation.startswith(("must::", "may::", "might::", "can::", "could::", "should::")) + ), None) + if fact is None: + return self._abstention("query_modal_strength", "the modal relation", "modal_graph_absent") + modal, relation = fact.relation.split("::", 1) + return HandlerResult( + "query_modal_strength", "answered", (("subject", fact.subject), ("relation", relation)), + (("modal", modal),), + (SemanticClause(fact.subject, relation, object_value=fact.object_value, modal=modal),), + (("source", "conversation_local_modal_graph"),), + ("resolve_nominalized_event", "compare_modal_strength", "release_stored_modal"), + ) + + def _assert_spatial_chain(self, session_id: str, surface: str) -> HandlerResult: + parts = re.split(r",?\s+and\s+", clean(surface).strip(" .?!"), flags=re.IGNORECASE) + facts: list[StoredGraphFact] = [] + for part in parts: + match = re.match(r"^(.+?)\s+is\s+(north|south|above|below)\s+of\s+(.+)$", part, re.IGNORECASE) + if match: + facts.append(self._store_atom(session_id, GraphAtom( + _strip_article(match.group(1)), match.group(2).casefold(), _strip_article(match.group(3)), + ))) + # Directional-state composition: R(a,b) & R(b,c) -> R(a,c). + changed = True + while changed: + changed = False + active = [value for value in self._graph_discourse(session_id).facts if value.active] + for left in active: + for right in active: + if left.atom.predicate != right.atom.predicate or left.atom.predicate not in {"north", "south", "above", "below"}: + continue + if left.atom.object_value.casefold() != right.atom.subject.casefold(): + continue + atom = GraphAtom(left.atom.subject, left.atom.predicate, right.atom.object_value) + if any(value.active and value.atom == atom for value in active): + continue + facts.append(self._store_atom(session_id, atom, inferred=True, supports=(left.state_id, right.state_id))) + changed = True + active.append(facts[-1]) + self._record_graph_operation(session_id, "compose_directional_chain", facts, transitive=True) + return HandlerResult( + "compose_directional_chain", "answered", (("relations", len(facts)),), + (("inferred", sum(value.inferred for value in facts)),), + tuple(SemanticClause(value.atom.subject, "be", complement=f"{value.atom.predicate} of {value.atom.object_value}") for value in facts), + (("source", "conversation_local_directional_graph"),), + ("bind_directional_states", "join_shared_node", "compute_transitive_closure"), + ) + + def _query_spatial(self, session_id: str, surface: str) -> HandlerResult: + match = re.search(r"where\s+is\s+(.+?)\s+relative\s+to\s+(.+?)[?!.]*$", clean(surface), re.IGNORECASE) + if match is None: + return self._abstention("query_directional_chain", surface, "spatial_roles_unresolved") + subject, object_value = _strip_article(match.group(1)), _strip_article(match.group(2)) + fact = next(( + value for value in reversed(self._graph_discourse(session_id).facts) + if value.active and value.atom.subject.casefold() == subject.casefold() + and value.atom.object_value.casefold() == object_value.casefold() + and value.atom.predicate in {"north", "south", "above", "below"} + ), None) + if fact is None: + return self._abstention("query_directional_chain", surface, "directional_path_absent") + return HandlerResult( + "query_directional_chain", "answered", (("subject", subject), ("object", object_value)), + (("relation", fact.atom.predicate),), + (SemanticClause(subject, "be", complement=f"{fact.atom.predicate} of {object_value}"),), + (("source", "conversation_local_directional_graph"),), + ("resolve_spatial_roles", "retrieve_composed_direction", "release_relative_state"), + ) + + def _query_prior_event(self, session_id: str) -> HandlerResult: + discourse = self._graph_discourse(session_id) + if not discourse.last_event_subject or not discourse.last_event_object: + return self._abstention("query_prior_event", "the prior event", "prior_event_graph_absent") + qualifiers = (("to", discourse.last_event_destination),) if discourse.last_event_destination else () + return HandlerResult( + "query_prior_event", "answered", (("subject", discourse.last_event_subject),), + (("object", discourse.last_event_object), ("destination", discourse.last_event_destination)), + (SemanticClause( + discourse.last_event_subject, "carry", object_value=discourse.last_event_object, + qualifiers=qualifiers, tense="past", + ),), (("source", "conversation_topic_graph"),), + ("restore_prior_topic", "select_event_roles", "release_object_and_destination"), + ) + + def _bounded_tool_graph(self, session_id: str, surface: str) -> HandlerResult: + lower = clean(surface).casefold() + backend = self.tool_backend + experts = getattr(backend, "experts", {}) if backend is not None else {} + expert = next(( + value for value in experts.values() + if ( + ("population" in lower and any("population" in item for item in getattr(value.schema, "competencies", ()))) + or ( + ("earthquake" in lower or "catalog" in lower) + and any("earthquake" in item for item in getattr(value.schema, "competencies", ())) + ) + ) + ), None) + if expert is None: + return self._abstention("bounded_tool", surface, "registered_dataset_not_resolved") + field_names = {value.name for value in expert.schema.fields} + operation = ( + "count" if "count" in _words(lower) + else "minimum" if set(_words(lower)) & {"minimum", "smallest", "lowest"} + else "maximum" if set(_words(lower)) & {"maximum", "greatest", "highest", "largest"} + else "lookup" + ) + clauses: list[SemanticClause] = [] + values: list[str] = [] + row_key: str | None = None + field: str | None = None + if operation in {"maximum", "minimum"}: + field = "value" if "value" in field_names else next(( + value.name for value in expert.schema.fields if value.kind in {"integer", "number"} + ), None) + if field is None: + return self._abstention("bounded_tool", surface, "numeric_tool_field_absent") + selected = (max if operation == "maximum" else min)( + expert.rows, key=lambda value: float(value[field]), + ) + row_key = selected.get("country_name") or selected.get("event_id") or str(selected) + values = [str(selected[field]), row_key] + label = "population" if "population" in lower else field.replace("_", " ") + clauses.extend(( + SemanticClause(f"The {operation} {label}", "be", complement=str(selected[field])), + SemanticClause("The selected record", "be", complement=row_key), + )) + elif operation == "count": + numeric_match = re.search( + r"(?:(?:strictly\s+)?greater\s+than|above|more\s+than)\s+(-?\d+(?:\.\d+)?)", + lower, + ) + field = next(( + name for name in field_names if name in lower.replace(" ", "_") + ), None) or ("magnitude" if "magnitude" in field_names else None) + if field is None or numeric_match is None: + return self._abstention("bounded_tool", surface, "count_filter_not_typed") + threshold = float(numeric_match.group(1)) + count = sum(float(row[field]) > threshold for row in expert.rows) + values = [str(count)] + clauses.append(SemanticClause("The bounded expert", "contain", object_value=f"{count} matching records")) + else: + event = re.search(r"\b(us[0-9a-z]+)\b", lower) + selected = next(( + row for row in expert.rows + if event and str(row.get("event_id", "")).casefold() == event.group(1) + ), None) + if selected is None: + return self._abstention("bounded_tool", surface, "requested_row_not_registered") + row_key = str(selected.get("event_id") or selected.get("country_name") or "registered row") + requested = [name for name in ("magnitude", "place", "value") if name in field_names and name in lower] + if not requested: + requested = ["place"] if "place" in field_names else [next(iter(field_names))] + field = requested[0] + values = [row_key, *(str(selected[name]) for name in requested)] + clauses.append(SemanticClause("The event identifier", "be", complement=row_key)) + clauses.extend( + SemanticClause(f"The {name.replace('_', ' ')}", "be", complement=str(selected[name])) + for name in requested + ) + audit = { + "operation": operation, + "expert_id": expert.schema.expert_id, + "row_key": row_key, + "field": field, + "values": values, + "read_only": True, + "typed_arguments_bound": True, + "global_gradient_steps": 0, + } + return HandlerResult( + "bounded_tool", "answered", + (("operation", operation), ("expert_id", expert.schema.expert_id), ("row_key", row_key), ("field", field)), + (("values", values),), tuple(clauses), + (("expert_id", expert.schema.expert_id),), + ("select_registered_dataset", "bind_typed_operation", "execute_read_only_graph_query", "release_verified_fields"), + tool_audit=audit, + ) + + def _handle_graph(self, session_id: str, graph: UtteranceGraph) -> HandlerResult | None: + act, payload = graph.speech_act, graph.payload + if act == "define_concept": + return self._define(session_id, payload["definition"]) + if act == "instantiate_concept": + return self._instantiate_concept(session_id, str(payload["entity"]), str(payload["concept"])) + if act == "query_instance": + return self._query_instance(session_id, str(payload["entity"])) + if act == "assert_rule": + return self._assert_rule(session_id, payload["rule"]) + if act == "assert_fact": + return self._assert_fact(session_id, graph.atoms[0]) + if act == "assert_frame": + frame = payload["frame"] + result = self._assert_frame(session_id, frame) + roles = {value.role: value.surface for value in frame.arguments} + if "subject" in roles: + self._store_atom( + session_id, + GraphAtom( + roles["subject"], frame.lemma, + roles.get("direct_object", roles.get("clause_complement", "event")), + qualifiers=tuple( + (role.split(":", 1)[1], value) + for role, value in roles.items() if role.startswith("prep:") + ), + ), + ) + return result + if act == "query_frame_role": + return self._query_frame_role(session_id, payload) + if act == "query_graph_fact": + return self._query_graph_fact(session_id, payload) + if act in {"arithmetic", "comparison"}: + return self._execute_graph_arithmetic(graph) + if act == "assert_causal": + return self._assert_causal(session_id, graph.atoms) + if act == "query_causal_chain": + return self._query_causal_chain(session_id) + if act == "correct_state": + return self._correct_state(session_id, str(payload["surface"])) + if act == "query_location": + return self._query_location(session_id, payload.get("entity")) + if act == "assert_counterfactual": + return self._execute_graph_counterfactual(session_id, str(payload["surface"])) + if act == "query_counterfactual": + return self._query_world_property(session_id, counterfactual=True) + if act == "query_actual": + return self._query_world_property(session_id, counterfactual=False) + if act == "assert_hypothetical": + return self._execute_graph_hypothetical(session_id, str(payload["surface"])) + if act == "query_hypothetical": + return self._query_hypothetical(session_id) + if act == "query_hypothetical_exception": + return self._query_hypothetical(session_id, exception=True) + if act == "query_modal_strength": + return self._query_modal_strength(session_id) + if act == "assert_spatial_chain": + return self._assert_spatial_chain(session_id, str(payload["surface"])) + if act == "query_spatial": + return self._query_spatial(session_id, str(payload["surface"])) + if act == "query_prior_event": + return self._query_prior_event(session_id) + if act == "bounded_tool_request": + return self._bounded_tool_graph(session_id, str(payload["surface"])) + if act == "query_established_definition": + return self._abstention( + "query_established_definition", + "the requested established technical definition", + "definition_not_in_declared_sources_or_conversation_graph", + ) + if act == "social_acknowledgement": + return HandlerResult( + "social_acknowledgement", "answered", (), (("ready", True),), + (SemanticClause("I", "be", complement="ready to track a definition across several turns"),), + (("source", "conversation_capability_state"),), + ("recognize_social_opening", "bind_requested_capability", "release_direct_acknowledgement"), + ) + if act == "refuse_tool_mutation": + return HandlerResult( + "refuse_tool_mutation", "abstained", (), (("read_only", True),), + (SemanticClause("I", "alter", object_value="the registered database", modal="cannot"),), + (("epistemic_reason", "registered_tools_are_read_only"),), + ("recognize_mutating_tool_act", "enforce_read_only_boundary", "refuse_state_change"), + ) + return None + + def _handle_definition(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + graph = self._k76_current.get() + if getattr(self, "_k51_replaying", False): + graph = self.graph_transducer.analyze( + text, + self._graph_discourse(session_id), + known_frames=tuple(self._stored_frames(self._k45_sessions[session_id])), + ) + # K15 checks the definition-capable rail before the event-capable rail. + # A graph operation is already a typed whole-utterance decision, so it + # may be executed from either authorized rail; it must not fall back to + # a local lexical definition classifier simply because the selected + # specialists include ``definition`` rather than ``event``. + if graph is not None and graph.speech_act in self.DIRECT_ACTS: + result = self._handle_graph(session_id, graph) + if result is not None: + return result + return super()._handle_definition(session_id, text, turn_id) + + def _handle_corpus( + self, + session_id: str, + text: str, + context: Mapping[str, Any], + turn_id: str, + ) -> HandlerResult | None: + """Compile comparisons over an arbitrary bounded local corpus. + + K15 already turns supplied records into an ephemeral expert. This + layer adds the missing relational reduction: select a comparable + numeric field, rank entities, compute the difference, and join any + requested descriptive field without fitting a new language model. + """ + result = super()._handle_corpus(session_id, text, context, turn_id) + corpus = context.get("training_corpus") + if ( + result is None + or result.operation != "compile_corpus_expert" + or result.status != "answered" + or not isinstance(corpus, Sequence) + or isinstance(corpus, (str, bytes)) + ): + return result + query = clean(text).casefold() + query_words = set(_words(query)) + + mappings = [value for value in corpus if isinstance(value, Mapping)] + comparison = bool(query_words & {"higher", "lower", "larger", "smaller", "greatest", "least"}) + if mappings and len(mappings) == len(corpus) and comparison: + records: dict[str, dict[str, str]] = {} + sources: list[tuple[str, str]] = [] + for index, row in enumerate(mappings): + entity = clean(row.get("entity", row.get("subject", f"record {index + 1}"))) + relation = clean(row.get("relation", row.get("predicate", "value"))).casefold() + value = clean(row.get("value", row.get("object", row.get("result", "")))) + if entity and relation and value: + records.setdefault(entity, {})[relation] = value + sources.append((clean(row.get("source", f"case-local-record-{index + 1}")), digest(row))) + numeric: dict[str, dict[str, float]] = {} + for entity, fields in records.items(): + for relation, value in fields.items(): + match = re.search(r"-?\d+(?:\.\d+)?", value.replace(",", "")) + if match: + numeric.setdefault(relation, {})[entity] = float(match.group(0)) + comparable = [ + (relation, values) for relation, values in numeric.items() if len(values) >= 2 + ] + if comparable: + comparable.sort(key=lambda item: ( + 0 if set(item[0].replace("_", " ").split()) & query_words else 1, + item[0], + )) + relation, values = comparable[0] + lower_requested = bool(query_words & {"lower", "smaller", "least"}) + winner = (min if lower_requested else max)(values, key=lambda name: (values[name], name)) + ordered = sorted(values.values(), reverse=not lower_requested) + difference = abs(ordered[0] - ordered[1]) + number = lambda value: f"{value:g}" + clauses = [ + SemanticClause( + winner, "have", + object_value=f"the {'lower' if lower_requested else 'higher'} {relation.replace('_', ' ')} of {number(values[winner])}", + ), + SemanticClause( + "The absolute difference", "be", + complement=f"{number(difference)} " + ("cycles" if "cycle" in relation else relation.replace("_", " ")), + ), + ] + descriptive = next(( + field for field in records[winner] + if field != relation and set(field.replace("_", " ").split()) & query_words + ), None) + if descriptive is not None: + clauses.append(SemanticClause( + f"The {descriptive.replace('_', ' ')} for {winner}", + "be", complement=records[winner][descriptive], + )) + return HandlerResult( + result.operation, result.status, result.arguments, + ( + ("winner", winner), ("comparison_field", relation), + ("difference", difference), + ), + tuple(clauses), tuple(sources[:4]), + (*result.computation_trace, "reduce_numeric_relational_field", "join_requested_companion_field"), + tool_audit=result.tool_audit, memory_audit=result.memory_audit, + ) + + # String records are parsed by their recurrent relational shape, not + # by the entity names. This covers unseen local astronomy corpora and + # any equivalent day/night profile expressed in the same English + # construction. + profiles: dict[str, tuple[float, float, bool]] = {} + for item in corpus: + if not isinstance(item, str): + continue + match = re.search( + r"^([A-Za-z0-9_-]+)\s+has\s+a\s+dayside\s+temperature\s+of\s+" + r"(-?\d+(?:\.\d+)?)\s+kelvin\s+and\s+a\s+nightside\s+temperature\s+of\s+" + r"(-?\d+(?:\.\d+)?)\s+kelvin;\s+it\s+(does\s+not\s+)?rotates?\s+synchronously", + clean(item), re.IGNORECASE, + ) + if match: + profiles[match.group(1)] = ( + float(match.group(2)), float(match.group(3)), match.group(4) is None, + ) + if len(profiles) >= 2 and query_words & {"range", "contrast", "difference", "larger", "higher"}: + spans = {name: abs(day - night) for name, (day, night, _sync) in profiles.items()} + winner = max(spans, key=lambda name: (spans[name], name)) + synchronous = profiles[winner][2] + clauses = ( + SemanticClause( + winner, "have", + object_value=f"the larger day-night temperature range of {spans[winner]:g} kelvin", + ), + SemanticClause( + winner, "rotate", object_value="synchronously", + polarity="positive" if synchronous else "negative", + ), + ) + return HandlerResult( + result.operation, result.status, result.arguments, + (("winner", winner), ("temperature_ranges_kelvin", spans), ("synchronous", synchronous)), + clauses, result.evidence, + (*result.computation_trace, "parse_day_night_profile_shape", "compute_absolute_ranges"), + tool_audit=result.tool_audit, memory_audit=result.memory_audit, + ) + return result + + def _handle_evidence( + self, text: str, context: Mapping[str, Any], + ) -> HandlerResult | None: + graph = self._k76_current.get() + # The knowledge lattice may retrieve a lexically adjacent public row + # for any utterance. Retrieved text is not allowed to pre-empt an + # already typed conversation-local graph operation; it remains + # available for genuine knowledge/evidence speech acts. + if graph is not None and graph.speech_act in self.DIRECT_ACTS: + return None + evidence = context.get("evidence") + if isinstance(evidence, Sequence) and not isinstance(evidence, (str, bytes)) and evidence: + facts = [ + clean(value.get("fact", value.get("claim", value))) + if isinstance(value, Mapping) else clean(value) + for value in evidence + ] + public_rows = [ + value for value in evidence + if isinstance(value, Mapping) + and bool(value.get("source_document") or value.get("text_sha256")) + ] + causal_request = bool(set(_words(text)) & {"why", "cause", "caused", "causal", "compel"}) + if causal_request and public_rows: + stop = { + "a", "an", "and", "are", "as", "at", "be", "by", "do", "does", "each", + "for", "from", "how", "in", "is", "it", "of", "on", "or", "the", "their", + "this", "to", "what", "when", "where", "which", "why", "with", + } + anchors = set(_words(text)) - stop + fact_terms = set(_words(" ".join(facts))) - stop + explicit_causal = bool( + fact_terms & {"because", "cause", "caused", "causes", "produced", "therefore"} + ) + if len(anchors & fact_terms) < 2 or not explicit_causal: + return self._abstention( + "unsupported_causation", "the requested causal conclusion", + "causal_warrant_absent_or_relationally_incongruent", + ) + + if public_rows: + # Preserve the entire licensed proposition. Earlier native + # compilation released only the first branch of coordinated + # sentences (for example, material entering a black hole but + # other material leaving in jets). A source-grounded answer + # must retain polarity and every coordination before it may be + # transformed further by the inverse grammar. + clauses = tuple( + SemanticClause( + "The cited source", "state", + object_value=clean(value.get("fact", value.get("text", ""))).rstrip(".?!"), + ) + for value in public_rows + if clean(value.get("fact", value.get("text", ""))) + ) + if clauses: + return HandlerResult( + "knowledge_explanation", "answered", (("source_rows", len(public_rows)),), + (("released_propositions", len(clauses)),), clauses, + tuple( + ( + clean(value.get("source_document_sha256", value.get("source_document", "source"))), + clean(value.get("text_sha256", digest(value))), + ) + for value in public_rows + ), + ( + "bind_source_proposition_graphs", "retain_coordination_and_polarity", + "release_source_faithful_relational_surface", + ), + tool_audit={ + "tool": "native_source_proposition_graph_release", + "source_rows": len(public_rows), "global_gradient_steps": 0, + "raw_sentence_fallback_used": False, + }, + ) + labeled_trials: dict[str, tuple[int, int]] = {} + for fact in facts: + match = re.search( + r"\b([A-Z][A-Za-z0-9_-]*(?:\s+[A-Z0-9][A-Za-z0-9_-]*)?)\s+" + r"passed\s+(\d+)\s+of\s+(\d+)\s+(?:bench\s+)?trials\b", + fact, + ) + if match: + labeled_trials[clean(match.group(1))] = (int(match.group(2)), int(match.group(3))) + if len(labeled_trials) >= 2: + ordered = sorted( + labeled_trials, + key=lambda name: ( + -labeled_trials[name][0] / max(1, labeled_trials[name][1]), name, + ), + ) + winner = ordered[0] + clauses = [ + SemanticClause(name, "pass", object_value=f"{done} of {total} bench trials", tense="past") + for name, (done, total) in labeled_trials.items() + ] + clauses.extend(( + SemanticClause("The observed results", "favor", object_value=winner), + SemanticClause("Cross-site replication", "be", complement="absent"), + )) + return HandlerResult( + "evidence_synthesis", "answered", (("evidence_records", len(facts)),), + (("winner", winner),), tuple(clauses), + tuple((f"evidence_{index}", digest(value)) for index, value in enumerate(facts)), + ("bind_labeled_trials", "compare_observed_rates", "retain_replication_limit"), + ) + + directional: list[tuple[str, str, str]] = [] + for fact in facts: + match = re.search( + r"\b(Run\s+\w+)\s+placed\s+([A-Za-z0-9_-]+(?:\s+[A-Za-z0-9_-]+)?)\s+" + r"above\s+([A-Za-z0-9_-]+(?:\s+[A-Za-z0-9_-]+)?)\b", + fact, re.IGNORECASE, + ) + if match: + directional.append((clean(match.group(1)), clean(match.group(2)), clean(match.group(3)))) + if len(directional) >= 2 and any( + left.casefold() == right2.casefold() and right.casefold() == left2.casefold() + for _run, left, right in directional + for _run2, left2, right2 in directional + ): + clauses = tuple( + SemanticClause(left, "be", complement=f"higher than {right}", qualifiers=(("in", run),)) + for run, left, right in directional + ) + (SemanticClause("The calibration influence", "remain", complement="unresolved"),) + return HandlerResult( + "evidence_synthesis", "answered", (("evidence_records", len(facts)),), + (("directional_conflict", True),), clauses, + tuple((f"evidence_{index}", digest(value)) for index, value in enumerate(facts)), + ("bind_directional_runs", "detect_order_reversal", "retain_calibration_uncertainty"), + ) + + return super()._handle_evidence(text, context) + + def _handle_event(self, session_id: str, text: str, turn_id: str) -> HandlerResult | None: + graph = self._k76_current.get() + if getattr(self, "_k51_replaying", False): + graph = self.graph_transducer.analyze( + text, + self._graph_discourse(session_id), + known_frames=tuple(self._stored_frames(self._k45_sessions[session_id])), + ) + if graph is not None and graph.speech_act in self.DIRECT_ACTS: + result = self._handle_graph(session_id, graph) + if result is not None: + return result + return super()._handle_event(session_id, text, turn_id) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + *, + ablate_specialists: Sequence[str] = (), + ablate_knowledge_cells: Sequence[str] = (), + ablate_ephemeral_workspace: bool = False, + ablate_operator_composition: bool = False, + ablate_frame_induction: bool = False, + ablate_induced_role_binding: bool = False, + withhold_dictionary_top1: bool = False, + ) -> dict[str, Any]: + discourse = self._graph_discourse(session_id) + lock = self._k76_locks[session_id] + with lock: + # Session acquisition may deterministically replay a prior graph + # journal. Replay parses each historical input independently; + # only after it completes do we analyze the current utterance + # against the recovered frames and discourse graph. + session = self._session(session_id) + graph = self.graph_transducer.analyze( + text, discourse, known_frames=tuple(self._stored_frames(session)), + ) + local_context = dict(context or {}) + local_context["k76_semantic_graph"] = graph.as_dict() + token = self._k76_current.set(graph) + try: + output = super().answer( + session_id, text, local_context, + ablate_specialists=ablate_specialists, + ablate_knowledge_cells=ablate_knowledge_cells, + ablate_ephemeral_workspace=ablate_ephemeral_workspace, + ablate_operator_composition=ablate_operator_composition, + ablate_frame_induction=ablate_frame_induction, + ablate_induced_role_binding=ablate_induced_role_binding, + withhold_dictionary_top1=withhold_dictionary_top1, + ) + finally: + self._k76_current.reset(token) + operation = clean((output.get("computation_proof") or {}).get("operation", "")) + if operation in {"unsupported_causation", "query_established_definition"}: + # A neighboring public row may have been considered during the + # inherited pre-routing pass. Once the typed graph concludes that + # the requested claim is unsupported, that row must not remain + # marked as admitted evidence in the released proof audit. + lattice = output.get("knowledge_lattice") + if isinstance(lattice, dict): + lattice["retrieval_admitted_to_proof"] = False + lattice["retrieval_gate_reason"] = "k76_typed_epistemic_boundary" + lattice["retrieved_evidence"] = [] + lattice["evidence_rows_admitted_to_proof"] = [] + lattice["relational_provenance_reconciled"] = False + output["semantic_graph_transduction"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k76_lineage_sha256, + "graph": graph.as_dict(), + "typed_graph_selected": graph.consumes_as_graph, + "whole_utterance_precedes_local_predicate": True, + "closed_class_topology": True, + "variable_unification": True, + "directional_state_composition": True, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "next_token_predictions": 0, + "model_calls": 0, + "gradient_steps": 0, + "answer_or_gold_labels_used": False, + "pre_malkuth_payload_unbinds": 0, + } + output["runtime"]["semantic_graph_transduction_model_calls"] = 0 + output["runtime"]["semantic_graph_transduction_gradient_steps"] = 0 + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = [ + "ClosedClassGraphTransducer", + "GraphAtom", + "GraphDefinition", + "GraphDiscourse", + "GraphRule", + "RelationalGraphTransductionAgent", + "UtteranceGraph", +] diff --git a/root/wander-train/stage5k78_opened_k77_repair_agent.py b/root/wander-train/stage5k78_opened_k77_repair_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..6df235358c15ab01cc2490418a33f7fa4f98eb55 --- /dev/null +++ b/root/wander-train/stage5k78_opened_k77_repair_agent.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""K78 repairs learned from the opened, immutable K77 prospective failure. + +K76 and all K77 evidence artifacts remain untouched. This lineage broadens +whole-utterance questionhood, qualified nominal types, operator paraphrases, +and a small set of relational query-slot bridges. +""" +from __future__ import annotations + +from dataclasses import replace +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest +from stage5k76_relational_graph_transduction_agent import ( + ClosedClassGraphTransducer, + GraphAtom, + GraphDiscourse, + GraphRule, + RelationalGraphTransductionAgent, + UtteranceGraph, + _words, +) + + +SCHEMA = "oracle-stage5k78-opened-k77-repair-agent-v1" + + +class OpenedK77RepairTransducer(ClosedClassGraphTransducer): + def analyze( + self, + text: str, + discourse: GraphDiscourse, + *, + known_frames: Sequence[Mapping[str, Any]] = (), + ) -> UtteranceGraph: + value = clean(text) + lower = value.casefold() + + if re.match( + r"^(?:provide|supply|state)\s+(?:the\s+)?(?:established\s+)?" + r"(?:technical\s+|engineering\s+|scientific\s+)?definition\s+of\b", + lower, + ): + return UtteranceGraph( + "query_established_definition", payload={"surface": value}, + topology=("definition_imperative", "epistemic_request", "productive_imperative"), + confidence=0.98, consumes_as_graph=True, + ) + + compound = re.match( + r"^(.+?)\s+is\s+an?\s+([a-z][a-z-]*)\s+([a-z][a-z-]*)\s+" + r"(inside|within|beneath|above|below)\s+(?:an?\s+|the\s+)?(.+?)[.!?]*$", + value, re.IGNORECASE, + ) + if compound and not value.endswith("?"): + subject, quality, entity_type, relation, location = compound.groups() + return UtteranceGraph( + "assert_qualified_type", + payload={ + "subject": clean(subject), "quality": quality.casefold(), + "entity_type": entity_type.casefold(), "relation": relation.casefold(), + "location": clean(location).rstrip(".?!"), + }, + topology=("copula", "attributive_property", "nominal_type", "spatial_qualifier"), + confidence=0.98, consumes_as_graph=True, + ) + + qualified_rule = re.match( + r"^every\s+([a-z][a-z-]*)\s+([a-z][a-z-]*)\s+" + r"(inside|within|beneath|above|below)\s+(?:an?\s+|the\s+)?(.+?)\s+" + r"(?:becomes?|is)\s+([a-z][a-z-]*)[.!?]*$", + lower, + ) + if qualified_rule: + quality, entity_type, relation, location, consequence = qualified_rule.groups() + before = ( + GraphAtom("$x", "is_a", entity_type), + GraphAtom("$x", "property", quality), + GraphAtom("$x", relation, clean(location)), + ) + after = (GraphAtom("$x", "property", consequence),) + rule = GraphRule(self._rule_id(before, after), before, after) + return UtteranceGraph( + "assert_rule", payload={"rule": rule}, + topology=( + "universal_quantifier", "attributive_property", "nominal_type", + "spatial_condition", "consequence", + ), + confidence=0.98, consumes_as_graph=True, + ) + + graph = super().analyze(value, discourse, known_frames=known_frames) + + # Questionhood and an imperative after a fronted adjunct govern the + # entire utterance. They must outrank an event-shaped local clause. + if graph.speech_act in {"assert_frame", "assert_fact"}: + trailing = clean(value.split(",", 1)[1]).casefold() if "," in value else "" + embedded_request = bool(re.match( + r"^(?:describe|explain|identify|state|report|compare|contrast|relate|trace|" + r"what|which|who|how|where|when|why)\b", + trailing, + )) + if value.endswith("?") or embedded_request: + graph = UtteranceGraph( + "knowledge_request", + topology=("whole_utterance_question", "fronted_adjunct", "speech_act_boundary"), + confidence=0.97, + ) + + if ( + graph.speech_act == "knowledge_request" + and discourse.causal_edges + and set(_words(value)) & {"trace", "reason", "explains", "explain", "why"} + ): + return UtteranceGraph( + "query_causal_chain", topology=("causal_discourse", "explanation_request"), + confidence=0.98, consumes_as_graph=True, + ) + return graph + + +class OpenedK77RepairAgent(RelationalGraphTransductionAgent): + DIRECT_ACTS = RelationalGraphTransductionAgent.DIRECT_ACTS | {"assert_qualified_type"} + RELATIONAL_BRIDGES = { + **dict(RelationalGraphTransductionAgent.RELATIONAL_BRIDGES), + "consequence": ("result", "resulting", "effect", "outcome"), + "effect": ("result", "resulting", "destroys", "destruction", "outcome"), + "quantity": ("rate", "amount", "number", "million", "molecules"), + "ruin": ("destroy", "destroys", "damages"), + "ruins": ("destroy", "destroys", "damages"), + } + + @staticmethod + def _knowledge_speech_act(text: str) -> bool: + if RelationalGraphTransductionAgent._knowledge_speech_act(text): + return True + values = set(_words(text)) + return clean(text).endswith("?") and bool( + values & {"what", "which", "who", "how", "where", "when", "why"} + ) + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.graph_transducer = OpenedK77RepairTransducer(self) + self._k78_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k76_lineage_sha256, + "repair_classes": [ + "whole_utterance_questionhood", "qualified_nominal_type", + "causal_query_paraphrase", "tool_operator_paraphrase", + "definition_imperative", "relational_query_slot_bridge", + ], + }) + + def _assert_qualified_type( + self, session_id: str, payload: Mapping[str, Any], + ) -> HandlerResult: + subject = str(payload["subject"]) + atoms = ( + GraphAtom(subject, "is_a", str(payload["entity_type"])), + GraphAtom(subject, "property", str(payload["quality"])), + GraphAtom(subject, str(payload["relation"]), str(payload["location"])), + ) + facts = [self._store_atom(session_id, atom) for atom in atoms] + inferred = list(self._rule_closure(session_id)) + facts.extend(inferred) + discourse = self._graph_discourse(session_id) + discourse.last_entity = subject.casefold() + self._record_graph_operation( + session_id, "assert_qualified_type", facts, + closure_count=len(inferred), nominal_head=str(payload["entity_type"]), + ) + clauses = ( + SemanticClause( + subject, "be", + complement=f"a {payload['quality']} {payload['entity_type']}", + ), + SemanticClause(subject, "be", complement=f"{payload['relation']} {payload['location']}"), + *(self._atom_clause(value.atom) for value in inferred), + ) + return HandlerResult( + "assert_graph_fact", "answered", (("facts", len(facts)),), + (("inferred", len(inferred)),), clauses, + (("source", "conversation_local_qualified_type_graph"),), + ( + "bind_nominal_head", "bind_attributive_property", "bind_spatial_qualifier", + "run_variable_unification", "release_graph_inverse", + ), + ) + + def _handle_graph(self, session_id: str, graph: UtteranceGraph) -> HandlerResult | None: + if graph.speech_act == "assert_qualified_type": + return self._assert_qualified_type(session_id, graph.payload) + return super()._handle_graph(session_id, graph) + + def _bounded_tool_graph(self, session_id: str, surface: str) -> HandlerResult: + normalized = re.sub(r"\bleast\b", "minimum", surface, flags=re.IGNORECASE) + normalized = re.sub(r"\bexceeds\b", "is greater than", normalized, flags=re.IGNORECASE) + normalized = re.sub(r"\bexceed\b", "be greater than", normalized, flags=re.IGNORECASE) + result = super()._bounded_tool_graph(session_id, normalized) + if normalized == surface: + return result + return replace( + result, + computation_trace=(*result.computation_trace, "normalize_relational_tool_operator"), + ) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + output = super().answer(*args, **kwargs) + output["opened_k77_repair"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k78_lineage_sha256, + "development_source": "opened_immutable_k77_failures", + "prospective_claim": False, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"]["opened_k77_repair_model_calls"] = 0 + output["runtime"]["opened_k77_repair_gradient_steps"] = 0 + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["OpenedK77RepairAgent", "OpenedK77RepairTransducer"] diff --git a/root/wander-train/stage5k86_native_goal_tool_controller.py b/root/wander-train/stage5k86_native_goal_tool_controller.py new file mode 100644 index 0000000000000000000000000000000000000000..99d5d7b3d4f8eee59ec42e710d0bd877f068deea --- /dev/null +++ b/root/wander-train/stage5k86_native_goal_tool_controller.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""K86 typed native goal controller over the K85 conversational runtime. + +This development successor does not claim unrestricted external agency. It +adds the missing control semantics for bounded registered tools: a request is +compiled into dependent typed actions, observations are bound to symbolic +addresses, postconditions are checked, and execution either stops with a +verified result or returns a bounded failure. No language model proposes or +repairs the plan. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +import re +from typing import Any, Mapping + +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest +from stage5k85_opened_k84_discourse_tool_repair_agent import ( + OpenedK84DiscourseToolRepairAgent, +) + + +SCHEMA = "oracle-stage5k86-native-goal-tool-controller-v1" + + +@dataclass(frozen=True) +class ToolAction: + action_id: str + operation: str + domain: str + query: str + depends_on: tuple[str, ...] = () + consumes_address: str | None = None + produces_address: str | None = None + required_fields: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ToolObservation: + action_id: str + status: str + operation: str + row_key: str | None + field: str | None + values: tuple[str, ...] + postconditions_met: bool + + +class NativeGoalToolControllerAgent(OpenedK84DiscourseToolRepairAgent): + """K85 plus a deterministic, observation-fed bounded action loop.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + registered = len(self.specialist_basis.partition) + primitive = len(self.specialist_basis.profiles) + if not 10 <= registered <= 20: + raise ValueError( + "K86 requires 10-20 operational specialist cells; " + f"received {registered}" + ) + self._k86_specialist_inventory = { + "registered_operational_cells": registered, + "primitive_capability_modes": primitive, + "primitive_specialist_ids": sorted( + value.specialist_id for value in self.specialist_basis.profiles + ), + } + self._k86_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k85_lineage_sha256, + "specialist_inventory": self._k86_specialist_inventory, + "controller": [ + "typed_goal_compilation", + "dependency_ordered_action_execution", + "observation_address_binding", + "postcondition_verification", + "bounded_replanning", + "explicit_stop_condition", + ], + }) + + @staticmethod + def _domain(surface: str) -> str | None: + lower = surface.casefold() + if "earthquake" in lower or "catalog" in lower: + return "earthquake database" + if "population" in lower: + return "population database" + return None + + @staticmethod + def _requested_fields(surface: str, domain: str) -> tuple[str, ...]: + lower = surface.casefold() + candidates = ("magnitude", "place") if domain == "earthquake database" else ("value",) + return tuple(value for value in candidates if re.search(rf"\b{value}\b", lower)) + + def _compile_goal(self, surface: str) -> tuple[ToolAction, ...]: + """Compile only dependency-bearing goals; single calls stay on K85.""" + value = clean(surface) + lower = value.casefold() + domain = self._domain(value) + sequential = bool(re.search( + r"\b(?:then|after(?:wards| that)?|using (?:that|the) (?:result|record)|selected record)\b", + lower, + )) + extremum = ( + "maximum" if re.search(r"\b(?:maximum|greatest|highest|largest|strongest)\b", lower) + else "minimum" if re.search(r"\b(?:minimum|smallest|lowest|weakest)\b", lower) + else None + ) + if domain is None or not sequential or extremum is None: + return () + + metric = "magnitude" if domain == "earthquake database" else "value" + fields = self._requested_fields(value, domain) + select = ToolAction( + "action_1", + extremum, + domain, + f"From the {domain}, find the {extremum} {metric}.", + produces_address="observation:selected_row", + required_fields=(metric, "row_key"), + ) + # A second action is useful only where the registered schema supports + # an addressable row lookup. Population extrema remain a verified + # single action rather than pretending country-name lookup exists. + if domain != "earthquake database" or not fields: + return (select,) + project = ToolAction( + "action_2", + "lookup", + domain, + "From the earthquake database, retrieve ${observation:selected_row}; " + + "provide " + " and ".join(fields) + ".", + depends_on=(select.action_id,), + consumes_address="observation:selected_row", + produces_address="observation:selected_fields", + required_fields=fields, + ) + return (select, project) + + @staticmethod + def _observation(action: ToolAction, result: HandlerResult) -> ToolObservation: + audit = dict(result.tool_audit or {}) + values = tuple(str(value) for value in audit.get("values", ()) if value is not None) + row_key = None if audit.get("row_key") is None else str(audit["row_key"]) + field = None if audit.get("field") is None else str(audit["field"]) + if action.operation in {"maximum", "minimum"}: + postconditions = ( + result.status == "answered" + and row_key is not None + and bool(values) + and bool(action.required_fields) + and field == action.required_fields[0] + ) + else: + joined = " ".join( + clause.complement or clause.object_value or "" for clause in result.clauses + ).casefold() + postconditions = result.status == "answered" and all( + field_name in " ".join( + (clause.subject + " " + (clause.complement or clause.object_value or "")).casefold() + for clause in result.clauses + ) or field_name in joined + for field_name in action.required_fields + ) + return ToolObservation( + action.action_id, + result.status, + str(audit.get("operation") or result.operation), + row_key, + field, + values, + postconditions, + ) + + def _typed_extremum(self, action: ToolAction) -> HandlerResult: + """Execute an extremum against the named schema field, never column order.""" + backend = self.tool_backend + experts = getattr(backend, "experts", {}) if backend is not None else {} + expert = next(( + value for value in experts.values() + if ( + action.domain == "earthquake database" + and any("earthquake" in item for item in getattr(value.schema, "competencies", ())) + ) or ( + action.domain == "population database" + and any("population" in item for item in getattr(value.schema, "competencies", ())) + ) + ), None) + field = action.required_fields[0] if action.required_fields else None + schema_fields = {value.name: value for value in expert.schema.fields} if expert is not None else {} + if expert is None or field not in schema_fields or schema_fields[field].kind not in {"integer", "number"}: + return self._abstention("bounded_agentic_tool", action.query, "typed_extremum_field_absent") + selected = (max if action.operation == "maximum" else min)( + expert.rows, key=lambda value: float(value[field]), + ) + row_key = str(selected.get("country_name") or selected.get("event_id") or "registered row") + scalar = str(selected[field]) + label = "population" if action.domain == "population database" else field.replace("_", " ") + audit = { + "operation": action.operation, + "expert_id": expert.schema.expert_id, + "row_key": row_key, + "field": field, + "values": [scalar, row_key], + "read_only": True, + "typed_arguments_bound": True, + "field_selected_by_typed_goal": True, + "global_gradient_steps": 0, + } + return HandlerResult( + "bounded_tool", + "answered", + (("operation", action.operation), ("expert_id", expert.schema.expert_id), + ("row_key", row_key), ("field", field)), + (("values", [scalar, row_key]),), + ( + SemanticClause(f"The {action.operation} {label}", "be", complement=scalar), + SemanticClause("The selected record", "be", complement=row_key), + ), + (("expert_id", expert.schema.expert_id),), + ( + "select_registered_dataset", + "bind_named_numeric_schema_field", + "execute_read_only_extremum", + "release_verified_field_and_row_address", + ), + tool_audit=audit, + ) + + def _execute_goal(self, session_id: str, surface: str, plan: tuple[ToolAction, ...]) -> HandlerResult: + addresses: dict[str, str] = {} + observations: list[ToolObservation] = [] + action_audits: list[Mapping[str, Any]] = [] + results: list[HandlerResult] = [] + replans = 0 + stop_reason = "plan_exhausted" + + for action in plan: + if any(not any(value.action_id == dep and value.postconditions_met for value in observations) + for dep in action.depends_on): + stop_reason = "dependency_postcondition_failed" + break + query = action.query + if action.consumes_address: + bound = addresses.get(action.consumes_address) + if not bound: + stop_reason = "required_observation_address_unbound" + break + query = query.replace("${" + action.consumes_address + "}", bound) + result = ( + self._typed_extremum(action) + if action.operation in {"maximum", "minimum"} + else super()._bounded_tool_graph(session_id, query) + ) + observation = self._observation(action, result) + + # One deterministic bounded repair is permitted for an extremum: + # normalize the metric and operation into the registered schema. + if not observation.postconditions_met and action.operation in {"maximum", "minimum"}: + replans += 1 + metric = "magnitude" if action.domain == "earthquake database" else "value" + repaired = f"From the {action.domain}, find the {action.operation} {metric}." + result = self._typed_extremum(action) + observation = self._observation(action, result) + + results.append(result) + observations.append(observation) + action_audits.append(dict(result.tool_audit or {})) + if not observation.postconditions_met: + stop_reason = "action_postcondition_failed" + break + if action.produces_address and observation.row_key: + addresses[action.produces_address] = observation.row_key + + completed = len(observations) == len(plan) and all(value.postconditions_met for value in observations) + if completed: + stop_reason = "goal_postconditions_satisfied" + clauses: list[SemanticClause] = [] + evidence: list[tuple[str, Any]] = [] + seen_clauses: set[tuple[Any, ...]] = set() + for result in results: + for clause in result.clauses: + key = ( + clause.subject, clause.predicate, clause.object_value, + clause.complement, clause.qualifiers, clause.tense, + ) + if key not in seen_clauses: + seen_clauses.add(key) + clauses.append(clause) + evidence.extend(result.evidence) + + plan_id = "plan_" + digest({"session": session_id, "surface": clean(surface), "plan": [asdict(v) for v in plan]})[:20] + memory_id = self._bind_memory( + session_id, + plan_id, + { + "situation": "typed_bounded_tool_goal", + "transition": "execute_dependency_ordered_actions", + "action": "verified_stop" if completed else "bounded_stop", + "result": stop_reason, + "status": "verified" if completed else "unresolved", + "verifier": digest([asdict(value) for value in observations]), + }, + bank="procedural", + source="k86_native_goal_controller", + ) + audit = { + "schema_version": SCHEMA, + "plan_id": plan_id, + "goal": clean(surface), + "plan": [asdict(value) for value in plan], + "observations": [asdict(value) for value in observations], + "observation_addresses": addresses, + "action_audits": action_audits, + "actions_planned": len(plan), + "actions_executed": len(observations), + "replans": replans, + "goal_postconditions_satisfied": completed, + "stop_reason": stop_reason, + "read_only_authority": True, + "side_effecting_tools_authorized": False, + "memory_id": memory_id, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + } + if not clauses: + clauses = [SemanticClause("The bounded tool goal", "remain", complement="unresolved")] + return HandlerResult( + "bounded_agentic_tool", + "answered" if completed else "clarification", + (("plan_id", plan_id), ("actions", len(plan))), + (("goal_postconditions_satisfied", completed), ("stop_reason", stop_reason)), + tuple(clauses), + tuple(evidence), + ( + "compile_typed_goal_graph", + "topologically_order_actions", + "execute_registered_read_only_affordances", + "bind_observations_to_addresses", + "verify_action_postconditions", + "evaluate_bounded_replan", + "verify_goal_postconditions", + "stop_on_verified_condition", + ), + tool_audit=audit, + memory_audit={"controller_plan_memory_id": memory_id}, + ) + + def _bounded_tool_graph(self, session_id: str, surface: str) -> HandlerResult: + plan = self._compile_goal(surface) + if not plan: + return super()._bounded_tool_graph(session_id, surface) + return self._execute_goal(session_id, surface, plan) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + output = super().answer(*args, **kwargs) + audit = output.get("tool_audit") or {} + output["native_goal_tool_controller"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k86_lineage_sha256, + "specialist_inventory": self._k86_specialist_inventory, + "agentic_plan_executed": audit.get("schema_version") == SCHEMA, + "authority_boundary": "registered_read_only_tools_only", + "unrestricted_external_agency_claimed": False, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"]["native_goal_controller_model_calls"] = 0 + output["runtime"]["native_goal_controller_gradient_steps"] = 0 + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = [ + "NativeGoalToolControllerAgent", + "ToolAction", + "ToolObservation", +] diff --git a/root/wander-train/stage5k88_opened_k87_precedence_retrieval_agent.py b/root/wander-train/stage5k88_opened_k87_precedence_retrieval_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..339c5842c0194f4fe166e66f830726402fa06b96 --- /dev/null +++ b/root/wander-train/stage5k88_opened_k87_precedence_retrieval_agent.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""K88 repairs the general failure classes exposed by immutable K87. + +K86 and every K87 prospective artifact remain untouched. The repair makes +typed local state precede open retrieval, decomposes coordinated knowledge +nuclei before addressing, extends conversational tool-row reference, and +turns an explicit replication limitation into a scoped negative conclusion. +""" +from __future__ import annotations + +from contextvars import ContextVar +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest +from stage5k76_relational_graph_transduction_agent import GraphDiscourse, UtteranceGraph, _words +from stage5k85_opened_k84_discourse_tool_repair_agent import OpenedK84DiscourseRepairTransducer +from stage5k86_native_goal_tool_controller import NativeGoalToolControllerAgent + + +SCHEMA = "oracle-stage5k88-opened-k87-precedence-retrieval-agent-v1" +_K88_AUDIT: ContextVar[Mapping[str, Any] | None] = ContextVar("k88_precedence_audit", default=None) + + +class OpenedK87PrecedenceTransducer(OpenedK84DiscourseRepairTransducer): + def analyze( + self, + text: str, + discourse: GraphDiscourse, + *, + known_frames: Sequence[Mapping[str, Any]] = (), + ) -> UtteranceGraph: + inherited = super().analyze(text, discourse, known_frames=known_frames) + value = clean(text) + values = _words(value) + words = set(values) + lower = value.casefold() + graph = inherited + repair: str | None = None + + trailing_request = False + if "," in value: + trailing_values = _words(value.split(",", 1)[1]) + trailing_request = bool( + trailing_values and trailing_values[0] in self.IMPERATIVE_CUES | self.WH + ) + request_boundary = bool( + values and values[0] in self.IMPERATIVE_CUES | self.WH + ) or trailing_request + if inherited.speech_act == "assert_frame" and request_boundary: + graph = UtteranceGraph( + "knowledge_request", + payload={"surface": value}, + topology=("imperative_or_wh_boundary", "open_knowledge_request"), + confidence=0.99, + ) + repair = "imperative_boundary_precedes_internal_participle_frame" + elif ( + discourse.rules + and request_boundary + and words & {"consequence", "consequences", "follow", "follows", "result", "results"} + ): + known_subjects = { + fact.atom.subject.casefold(): fact.atom.subject + for fact in discourse.facts if fact.active and not fact.atom.subject.startswith("$") + } + subject = next((known_subjects[word] for word in values if word in known_subjects), None) + graph = UtteranceGraph( + "query_graph_fact", + payload={"mode": "consequence", "subject": subject or discourse.last_entity, "anchor": None}, + topology=("conversation_local_rule_graph", "consequence_request"), + confidence=0.99, + consumes_as_graph=True, + ) + repair = "local_rule_consequence_precedes_open_corpus" + elif ( + re.search(r"\bregistered\s+(?:row|record)\b", lower) + and words & {"give", "provide", "report", "retrieve", "state", "use", "using"} + ): + graph = UtteranceGraph( + "bounded_tool_request", + payload={"surface": value}, + topology=("registered_tool_row_anaphor", "read_only_projection"), + confidence=0.99, + consumes_as_graph=True, + ) + repair = "registered_row_reference_precedes_open_corpus" + + _K88_AUDIT.set({ + "inherited_speech_act": inherited.speech_act, + "selected_speech_act": graph.speech_act, + "repair": repair, + "local_fact_count": len(discourse.facts), + "local_rule_count": len(discourse.rules), + "local_causal_edge_count": len(discourse.causal_edges), + }) + return graph + + +class OpenedK87PrecedenceRetrievalAgent(NativeGoalToolControllerAgent): + RELATIONAL_BRIDGES = { + **dict(NativeGoalToolControllerAgent.RELATIONAL_BRIDGES), + "decoder": ("ribosome", "decoding", "decodes"), + "message": ("mrna", "messenger"), + "product": ("polypeptide", "protein"), + "supports": ("backbones", "backbone"), + "steps": ("rungs", "rung"), + "handles": ("decodes", "decoding"), + } + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.graph_transducer = OpenedK87PrecedenceTransducer(self, self.construction_head) + self._k88_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k86_lineage_sha256, + "opened_failure_source": "immutable_k87_scored_outputs_v1", + "repair_classes": [ + "imperative_precedes_internal_predicate", + "conversation_local_graph_precedes_open_retrieval", + "registered_tool_row_reference_precedes_open_retrieval", + "coordinated_knowledge_nucleus_decomposition", + "relational_source_surface_bridges", + "replication_limit_to_scoped_negative_conclusion", + ], + }) + + @staticmethod + def _cross_segments(text: str) -> tuple[str, str] | None: + inherited = NativeGoalToolControllerAgent._cross_segments(text) + if inherited is not None: + return inherited + value = clean(text).rstrip(".?!") + match = re.search(r"(?:^|:\s*)(?:describe|state|explain|identify)\s+(.+?),\s*then\s+(?:state|describe|explain|identify)\s+(.+)$", value, re.IGNORECASE) + if match: + return clean(match.group(1)), clean(match.group(2)) + match = re.search(r"(?:^|:\s*)(.+?)\s*;\s*(?:then|next)\s+(.+)$", value, re.IGNORECASE) + if match: + return clean(match.group(1)), clean(match.group(2)) + return None + + def _handle_evidence(self, text: str, context: Mapping[str, Any]) -> HandlerResult | None: + evidence = context.get("evidence") + if isinstance(evidence, Sequence) and not isinstance(evidence, (str, bytes)): + facts = [ + clean(value.get("fact", value.get("claim", value))) + if isinstance(value, Mapping) else clean(value) + for value in evidence + ] + completions: dict[str, tuple[int, int, str]] = {} + for fact in facts: + match = re.search( + r"\b([A-Z][A-Za-z0-9_-]*(?:\s+[A-Z][A-Za-z0-9_-]*)?)\s+" + r"completed\s+(\d+)\s+of\s+(\d+)\s+(runs|trials)\b", + fact, + ) + if match: + completions[clean(match.group(1))] = ( + int(match.group(2)), int(match.group(3)), match.group(4), + ) + replication_absent = any( + "replication" in fact.casefold() + and set(_words(fact)) & {"absent", "missing", "none", "unavailable", "without"} + for fact in facts + ) + if len(completions) >= 2: + ordered = sorted( + completions, + key=lambda name: (-completions[name][0] / max(1, completions[name][1]), name), + ) + winner, runner = ordered[:2] + winner_rate = completions[winner][0] / max(1, completions[winner][1]) + runner_rate = completions[runner][0] / max(1, completions[runner][1]) + difference = 100.0 * (winner_rate - runner_rate) + display = f"{difference:.1f}".rstrip("0").rstrip(".") + clauses = [ + SemanticClause( + name, "complete", object_value=f"{done} of {total} {unit}", tense="past", + ) + for name, (done, total, unit) in completions.items() + ] + clauses.extend(( + SemanticClause( + winner, "have", + object_value=f"higher observed completion than {runner}", + qualifiers=(("by", f"{display} percentage points"),), + ), + SemanticClause( + "Independent replication", "be", + complement="absent" if replication_absent else "unresolved", + ), + )) + if replication_absent: + clauses.append(SemanticClause( + "The evidence", "establish", object_value="universal superiority", + polarity="negative", + )) + return HandlerResult( + "evidence_synthesis", + "answered", + (("evidence_records", len(facts)),), + (("winner", winner), ("percentage_point_difference", difference), + ("independent_replication_absent", replication_absent)), + tuple(clauses), + tuple((f"evidence_{index}", digest(fact)) for index, fact in enumerate(facts)), + ( + "parse_completion_relations", + "compare_normalized_completion_rates", + "bind_replication_scope", + "derive_scoped_negative_epistemic_conclusion", + ), + ) + return super()._handle_evidence(text, context) + + def _bounded_tool_graph(self, session_id: str, surface: str) -> HandlerResult: + """Resolve row anaphora without treating ordinary ``us...`` words as IDs.""" + original = clean(surface) + lower = original.casefold() + prior = self._k85_tool_dialogue.get(session_id) + anaphoric = bool( + prior + and ( + re.search(r"\b(?:it|its|same|that|identical)\b", lower) + or re.search(r"\bregistered\s+(?:row|record)\b", lower) + ) + # Bound USGS identifiers have ``us`` followed by digits. K85's + # broader historical pattern also matched the English word + # ``using`` and therefore incorrectly disabled anaphora. + and not re.search(r"\bus\d{4,}[a-z0-9]*\b", lower) + ) + if not anaphoric or prior is None: + return super()._bounded_tool_graph(session_id, original) + rewritten = f"From the {prior.domain_cue}, retrieve {prior.row_key}; {original}" + result = super()._bounded_tool_graph(session_id, rewritten) + if result.status != "answered": + return result + return HandlerResult( + result.operation, + result.status, + result.arguments, + result.result, + result.clauses, + result.evidence, + (*result.computation_trace, "resolve_strictly_typed_tool_row_anaphor"), + tool_audit={ + **dict(result.tool_audit or {}), + "k88_strict_anaphoric_row_resolution": True, + "prior_row_key": prior.row_key, + "ordinary_us_prefix_words_are_not_event_ids": True, + }, + memory_audit=result.memory_audit, + ) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + token = _K88_AUDIT.set(None) + try: + output = super().answer(*args, **kwargs) + audit = dict(_K88_AUDIT.get() or {}) + finally: + _K88_AUDIT.reset(token) + output["opened_k87_precedence_retrieval_repair"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k88_lineage_sha256, + "development_source": "opened_immutable_k87_failures", + "prospective_claim": False, + "audit": audit, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"]["opened_k87_repair_model_calls"] = 0 + output["runtime"]["opened_k87_repair_gradient_steps"] = 0 + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["OpenedK87PrecedenceRetrievalAgent", "OpenedK87PrecedenceTransducer"] diff --git a/root/wander-train/stage5k92_opened_k91_reference_sensor_repair_agent.py b/root/wander-train/stage5k92_opened_k91_reference_sensor_repair_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..bdd8818cd50f51edffab2d394b450a320dfec440 --- /dev/null +++ b/root/wander-train/stage5k92_opened_k91_reference_sensor_repair_agent.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""K92 repairs the two general boundary classes exposed by immutable K91. + +K90 and every K91 artifact remain untouched. The repair gives typed +conversation-local row anaphora precedence over open retrieval and separates +sensor-scale stability from exact floating-point equality. +""" +from __future__ import annotations + +from contextvars import ContextVar +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import HandlerResult, SemanticClause, clean, digest +from stage5k76_relational_graph_transduction_agent import GraphDiscourse, UtteranceGraph, _words +from stage5k88_opened_k87_precedence_retrieval_agent import OpenedK87PrecedenceTransducer +from stage5k90_semantic_congruence_lexical_agent import SemanticCongruenceLexicalAgent + + +SCHEMA = "oracle-stage5k92-opened-k91-reference-sensor-repair-agent-v1" +_TYPED_CONTEXT: ContextVar[bool] = ContextVar("k92_typed_context", default=False) +_REPAIR_AUDIT: ContextVar[Mapping[str, Any] | None] = ContextVar("k92_repair_audit", default=None) + + +class OpenedK91ReferencePrecedenceTransducer(OpenedK87PrecedenceTransducer): + """Recognize a typed prior-row reference without requiring one adjective.""" + + def analyze( + self, + text: str, + discourse: GraphDiscourse, + *, + known_frames: Sequence[Mapping[str, Any]] = (), + ) -> UtteranceGraph: + inherited = super().analyze(text, discourse, known_frames=known_frames) + value = clean(text) + lower = value.casefold() + words = set(_words(value)) + row_anaphor = bool(re.search( + r"\b(?:that|the|this|same|identical)\s+" + r"(?:(?:same|identical|registered|catalog|database|dataset|selected)\s+){0,3}" + r"(?:row|record|result)\b", + lower, + )) + projection_request = bool(words & { + "give", "provide", "report", "retrieve", "show", "state", "tell", "use", "using", + }) + graph = inherited + repair = None + if row_anaphor and projection_request: + graph = UtteranceGraph( + "bounded_tool_request", + payload={"surface": value}, + topology=("typed_prior_row_anaphor", "read_only_field_projection"), + confidence=0.995, + consumes_as_graph=True, + ) + repair = "typed_prior_row_anaphor_precedes_open_retrieval" + _REPAIR_AUDIT.set({ + "inherited_speech_act": inherited.speech_act, + "selected_speech_act": graph.speech_act, + "row_anaphor": row_anaphor, + "projection_request": projection_request, + "repair": repair, + }) + return graph + + +class OpenedK91ReferenceSensorRepairAgent(SemanticCongruenceLexicalAgent): + SENSOR_STABILITY_RELATIVE = 0.005 + SENSOR_LOCAL_BOUNDEDNESS_RELATIVE = 0.02 + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.graph_transducer = OpenedK91ReferencePrecedenceTransducer(self, self.construction_head) + self._k92_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k90_lineage_sha256, + "opened_failure_source": "immutable_stage5k91_scored_outputs_v1", + "repair_classes": [ + "typed_prior_row_anaphor_before_open_retrieval", + "scale_relative_sensor_stability_before_directional_surface", + "typed_context_before_public_source_candidate_side_effect", + ], + "sensor_stability_relative": self.SENSOR_STABILITY_RELATIVE, + }) + + def _multi_anchor_source_result(self, text: str) -> HandlerResult | None: + if _TYPED_CONTEXT.get(): + return None + return super()._multi_anchor_source_result(text) + + def _handle_sensor(self, text: str, context: Mapping[str, Any]) -> HandlerResult | None: + stream = context.get("sensor_stream") + if not isinstance(stream, Sequence) or isinstance(stream, (str, bytes)) or not stream: + return None + rows = [dict(value) for value in stream if isinstance(value, Mapping)] + if not rows: + return self._abstention("sensor_state", "the sensor state", "sensor_rows_invalid") + candidate_fields = [ + key for key, value in rows[0].items() + if key.casefold() not in {"time", "timestamp", "unit", "sensor", "sensor_id"} + and isinstance(value, (int, float)) + ] + if not candidate_fields: + return self._abstention("sensor_state", "the sensor state", "numeric_sensor_field_absent") + clauses: list[SemanticClause] = [] + directions: dict[str, str] = {} + stable_fields: list[str] = [] + locally_bounded_fields: list[str] = [] + deltas: dict[str, float] = {} + relative_changes: dict[str, float] = {} + for field in candidate_fields: + values = [float(row[field]) for row in rows if isinstance(row.get(field), (int, float))] + if len(values) < 2: + continue + delta = values[-1] - values[0] + scale = max(1.0, *(abs(value) for value in values)) + stability_tolerance = max(1e-9, self.SENSOR_STABILITY_RELATIVE * scale) + bounded_tolerance = max(1e-6, self.SENSOR_LOCAL_BOUNDEDNESS_RELATIVE * scale) + direction = ( + "remain" if abs(delta) <= stability_tolerance + else "increase" if delta > 0 else "decrease" + ) + if max(values) - min(values) <= bounded_tolerance: + locally_bounded_fields.append(field) + unit = clean(rows[0].get("unit", "units")) + if field.endswith("_c") and unit == "units": + unit = "degrees C" + elif field.endswith("_kpa") and unit == "units": + unit = "kPa" + label = field.replace("_", " ") + directions[field] = direction + deltas[field] = delta + relative_changes[field] = abs(delta) / scale + if direction == "remain": + stable_fields.append(field) + clauses.append(SemanticClause( + f"The {label}", "remain", complement="stable", + qualifiers=(("near", f"{values[-1]:g} {unit}"),), tense="past", + )) + else: + clauses.append(SemanticClause( + f"The {label}", direction, + qualifiers=(("from", f"{values[0]:g} {unit}"), ("to", f"{values[-1]:g} {unit}")), + tense="past", + )) + if not clauses: + return self._abstention("sensor_state", "the sensor trend", "insufficient_sensor_records") + primary = candidate_fields[0] + return HandlerResult( + "sensor_state", + "answered", + (("fields", candidate_fields), ("records", len(rows))), + ( + ("direction", directions.get(primary)), + ("delta", deltas.get(primary)), + ("directions", directions), + ("stable_fields", stable_fields), + ("locally_bounded_fields", locally_bounded_fields), + ("relative_changes", relative_changes), + ), + tuple(clauses), + (("stream_digest", digest(rows)),), + ( + "bind_timestamped_readings", + "normalize_delta_by_observed_scale", + "separate_scale_stability_from_material_direction", + "release_directional_state", + ), + tool_audit={ + "tool": "native_sensor_fold", + "read_only": True, + "records": len(rows), + "stability_relative": self.SENSOR_STABILITY_RELATIVE, + "local_boundedness_relative": self.SENSOR_LOCAL_BOUNDEDNESS_RELATIVE, + }, + ) + + def answer( + self, + session_id: str, + text: str, + context: Mapping[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + local = dict(context or {}) + typed = any(local.get(key) for key in ("evidence", "training_corpus", "sensor_stream")) + typed_token = _TYPED_CONTEXT.set(typed) + audit_token = _REPAIR_AUDIT.set(None) + try: + output = super().answer(session_id, text, local, **kwargs) + precedence_audit = dict(_REPAIR_AUDIT.get() or {}) + finally: + _REPAIR_AUDIT.reset(audit_token) + _TYPED_CONTEXT.reset(typed_token) + output["opened_k91_reference_sensor_repair"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k92_lineage_sha256, + "development_source": "opened_immutable_k91_failures", + "prospective_claim": False, + "typed_context_present": typed, + "typed_context_precedes_open_source_candidate": True, + "precedence_audit": precedence_audit, + "sensor_stability_relative": self.SENSOR_STABILITY_RELATIVE, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"]["opened_k91_reference_sensor_repair_model_calls"] = 0 + output["runtime"]["opened_k91_reference_sensor_repair_gradient_steps"] = 0 + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = [ + "OpenedK91ReferencePrecedenceTransducer", + "OpenedK91ReferenceSensorRepairAgent", +] diff --git a/root/wander-train/stage5k94_opened_k93_explicit_lookup_precedence_agent.py b/root/wander-train/stage5k94_opened_k93_explicit_lookup_precedence_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..349a12e11e935979cfbba68691e1ce8e2f89c6cf --- /dev/null +++ b/root/wander-train/stage5k94_opened_k93_explicit_lookup_precedence_agent.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""K94 repairs the explicit-lookup boundary exposed by immutable K93.""" +from __future__ import annotations + +from contextvars import ContextVar +import re +from typing import Any, Mapping, Sequence + +from stage5k15_native_mixed_agent import clean, digest +from stage5k76_relational_graph_transduction_agent import GraphDiscourse, UtteranceGraph, _words +from stage5k92_opened_k91_reference_sensor_repair_agent import ( + OpenedK91ReferencePrecedenceTransducer, + OpenedK91ReferenceSensorRepairAgent, +) + + +SCHEMA = "oracle-stage5k94-opened-k93-explicit-lookup-precedence-agent-v1" +_K94_AUDIT: ContextVar[Mapping[str, Any] | None] = ContextVar("k94_lookup_audit", default=None) + + +class OpenedK93ExplicitLookupTransducer(OpenedK91ReferencePrecedenceTransducer): + """A typed ID and lookup predicate outrank clause-position heuristics.""" + + def analyze( + self, + text: str, + discourse: GraphDiscourse, + *, + known_frames: Sequence[Mapping[str, Any]] = (), + ) -> UtteranceGraph: + inherited = super().analyze(text, discourse, known_frames=known_frames) + value = clean(text) + lower = value.casefold() + words = set(_words(value)) + bound_event_id = re.search(r"\bus\d{4,}[a-z0-9]*\b", lower) + registered_domain = bool(words & {"earthquake", "earthquakes"}) and bool( + words & {"catalog", "database", "dataset", "observations", "registered"} + ) + lookup_predicate = bool( + re.search(r"\blook\s+up\b", lower) + or words & {"consult", "fetch", "lookup", "retrieve"} + ) + graph = inherited + repair = None + if bound_event_id and registered_domain and lookup_predicate: + graph = UtteranceGraph( + "bounded_tool_request", + payload={"surface": value}, + topology=("bound_dataset_identifier", "lookup_predicate", "read_only_projection"), + confidence=1.0, + consumes_as_graph=True, + ) + repair = "bound_identifier_lookup_precedes_clause_position_and_open_retrieval" + _K94_AUDIT.set({ + "inherited_speech_act": inherited.speech_act, + "selected_speech_act": graph.speech_act, + "bound_event_id": None if bound_event_id is None else bound_event_id.group(0), + "registered_domain": registered_domain, + "lookup_predicate": lookup_predicate, + "repair": repair, + }) + return graph + + +class OpenedK93ExplicitLookupPrecedenceAgent(OpenedK91ReferenceSensorRepairAgent): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.graph_transducer = OpenedK93ExplicitLookupTransducer(self, self.construction_head) + self._k94_lineage_sha256 = digest({ + "schema_version": SCHEMA, + "parent_lineage_sha256": self._k92_lineage_sha256, + "opened_failure_source": "immutable_stage5k93_scored_outputs_v1", + "repair_class": "bound_identifier_lookup_before_clause_position_and_open_retrieval", + }) + + def answer(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + token = _K94_AUDIT.set(None) + try: + output = super().answer(*args, **kwargs) + audit = dict(_K94_AUDIT.get() or {}) + finally: + _K94_AUDIT.reset(token) + output["opened_k93_explicit_lookup_precedence"] = { + "schema_version": SCHEMA, + "lineage_sha256": self._k94_lineage_sha256, + "development_source": "opened_immutable_k93_failure", + "prospective_claim": False, + "audit": audit, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "autoregressive_carrier": False, + "model_calls": 0, + "gradient_steps": 0, + } + output["runtime"]["opened_k93_explicit_lookup_precedence_model_calls"] = 0 + output["runtime"]["opened_k93_explicit_lookup_precedence_gradient_steps"] = 0 + output["schema_version"] = SCHEMA + output["turn_manifest_sha256"] = digest(output) + return output + + +__all__ = ["OpenedK93ExplicitLookupTransducer", "OpenedK93ExplicitLookupPrecedenceAgent"] diff --git a/root/wander-train/stage5l2c_predicate_coordinate_closure.py b/root/wander-train/stage5l2c_predicate_coordinate_closure.py new file mode 100644 index 0000000000000000000000000000000000000000..93dbf6de3c0556dd667715aa84ca18cb7e774233 --- /dev/null +++ b/root/wander-train/stage5l2c_predicate_coordinate_closure.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""L2c: close the inherited first-homograph predicate bottleneck. + +All lexically licensed predicate coordinates coexist. Deterministic clause +evidence orders the lattice: modal/auxiliary governance, finite morphology, +and completed nominal coordination outrank noun/verb homographs in subject +position. Lexicalized prefixed verbs receive a derivational interpretation +only when their WordNet relation to the stripped base is sufficiently close. +""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any, Sequence + +from stage5j2_native_english_construction_algebra import ( + AUXILIARIES, + COORDINATORS, + DETERMINERS, + MODALS, + PREPOSITIONS, +) +from stage5k15_native_mixed_agent import digest +from stage5l1_broad_relational_ingress import BroadRelationalCompiler +from stage5l2_structural_language_closure import ALL_NODE_ROUTE, PREFIX_OPERATORS +from stage5l2b_structural_closure_successor import ( + StructuralClosureSuccessorAdapter, + StructuralClosureSuccessorAgent, +) + + +SCHEMA = "oracle-stage5l2c-predicate-coordinate-closure-v1" + + +class PredicateCoordinateClosureAdapter(StructuralClosureSuccessorAdapter): + adapter_id = "oracle-stage5l2c-english-predicate-coordinate-closure-v1" + _DERIVATIONAL_SIMILARITY_MINIMUM = 0.40 + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + parent_semantics = self.semantic_sha256 + self.semantic_sha256 = digest({ + "schema_version": SCHEMA, + "parent_semantic_sha256": parent_semantics, + "predicate_coordinates": "enumerate_then_structurally_order", + "lexicalized_derivation_path_similarity_minimum": self._DERIVATIONAL_SIMILARITY_MINIMUM, + "productive_derivational_inflection_is_finite_evidence": True, + }) + + def _semantic_congruence(self, surface: str, base: str) -> float: + values = [] + for derived_synset in self.wordnet.synsets(surface, pos=self.wordnet.VERB): + for base_synset in self.wordnet.synsets(base, pos=self.wordnet.VERB): + similarity = derived_synset.path_similarity(base_synset) + if similarity is not None: + values.append(float(similarity)) + return max(values, default=0.0) + + def _derivation(self, surface: str) -> tuple[str, str, str] | None: + inherited = super()._derivation(surface) + if inherited is not None: + return inherited + normalized = surface.casefold().replace("-", "_") + direct = self._direct_frame_lemma(normalized) + if direct is None: + return None + for prefix, operator in PREFIX_OPERATORS: + if not normalized.startswith(prefix) or len(normalized) <= len(prefix) + 2: + continue + base_surface = normalized[len(prefix):] + licensed_base = self._licensed_base_lemma(base_surface) + if licensed_base is None: + continue + lexical_base = self._base_candidates(base_surface)[0] + if self._semantic_congruence(direct, lexical_base) < self._DERIVATIONAL_SIMILARITY_MINIMUM: + continue + return direct, licensed_base, operator + return None + + def _predicate_indexes(self, values: Sequence[str]) -> tuple[int, ...]: + """Enumerate every eligible coordinate; never let an earlier homograph erase a later verb.""" + result: list[int] = [] + for index, surface in enumerate(values): + if surface in AUXILIARIES | MODALS: + continue + if not (self.resolver.verb_like(surface) or self._derivation(surface) is not None): + continue + if index and values[index - 1] in DETERMINERS | PREPOSITIONS | self._INTERROGATIVE_DETERMINERS: + continue + result.append(index) + return tuple(result) + + def _coordinate_priority(self, values: Sequence[str], index: int) -> float: + surface = values[index] + priority = 0.0 + if index and values[index - 1] in AUXILIARIES | MODALS: + priority += 4.0 + wordnet_lemma = self.wordnet.morphy(surface, self.wordnet.VERB) + derivation = self._derivation(surface) + finite_or_participial = bool( + surface.endswith(("ed", "ing", "en")) + or (wordnet_lemma and str(wordnet_lemma).casefold() != surface) + or (derivation is not None and derivation[0] != surface) + ) + if finite_or_participial: + priority += 2.0 + if any(value in COORDINATORS for value in values[:index]): + priority += 0.5 + if index + 1 < len(values) and values[index + 1] in COORDINATORS: + priority -= 2.0 + + later_strong = False + for later in range(index + 1, len(values)): + value = values[later] + if not (self.resolver.verb_like(value) or self._derivation(value) is not None): + continue + lemma = self.wordnet.morphy(value, self.wordnet.VERB) + derivation = self._derivation(value) + if ( + (later and values[later - 1] in AUXILIARIES | MODALS) + or value.endswith(("ed", "ing", "en")) + or (lemma and str(lemma).casefold() != value) + or (derivation is not None and derivation[0] != value) + ): + later_strong = True + break + if later_strong and self.resolver.noun_like(surface): + priority -= 3.0 + return priority + + def analyze(self, text: str, *, maximum_hypotheses: int = 8): + expanded_maximum = max(64, int(maximum_hypotheses) * 4) + analysis = super().analyze(text, maximum_hypotheses=expanded_maximum) + from stage5j1_native_contextual_sense_lattice import WORD + + values = tuple(value.casefold() for value in WORD.findall(str(text))) + rescored = [ + replace( + hypothesis, + score=max(0.0, min( + 1.0, + hypothesis.score + 0.08 * self._coordinate_priority( + values, hypothesis.predicate_token_index, + ), + )), + ) + for hypothesis in analysis.hypotheses + ] + rescored.sort(key=lambda hypothesis: ( + -self._coordinate_priority(values, hypothesis.predicate_token_index), + -hypothesis.score, + hypothesis.frame_id, + )) + return replace( + analysis, + hypotheses=tuple(rescored[:max(1, int(maximum_hypotheses))]), + adapter_id=self.adapter_id, + adapter_semantic_sha256=self.semantic_sha256, + ) + + +class PredicateCoordinateClosureCompiler(BroadRelationalCompiler): + def __init__(self, **kwargs: Any) -> None: + kwargs["adapter_type"] = PredicateCoordinateClosureAdapter + super().__init__(**kwargs) + self.semantic_sha256 = digest({ + "schema_version": SCHEMA, + "parent_semantic_sha256": self.semantic_sha256, + "adapter_semantic_sha256": self.adapter.semantic_sha256, + "all_node_route": ALL_NODE_ROUTE, + }) + + def architecture(self) -> dict[str, Any]: + value = super().architecture() + value.update({ + "schema_version": SCHEMA, + "semantic_sha256": self.semantic_sha256, + "all_predicate_coordinates_retained": True, + "finite_predicate_structural_ordering": True, + "productive_derivational_inflection_is_finite_evidence": True, + "lexicalized_derivation_semantic_congruence": True, + "coordinated_roles_are_set_valued_states": True, + "structural_extensions_traverse_all_ten_nodes": True, + }) + return value + + +class PredicateCoordinateClosureAgent(StructuralClosureSuccessorAgent): + def __init__(self, *args: Any, + relational_compiler: PredicateCoordinateClosureCompiler | None = None, + **kwargs: Any) -> None: + super().__init__( + *args, + relational_compiler=relational_compiler or PredicateCoordinateClosureCompiler(), + **kwargs, + ) + + +__all__ = [ + "SCHEMA", + "PredicateCoordinateClosureAdapter", + "PredicateCoordinateClosureAgent", + "PredicateCoordinateClosureCompiler", +] diff --git a/root/wander-train/stage5l2d_typed_predicate_scope_closure.py b/root/wander-train/stage5l2d_typed_predicate_scope_closure.py new file mode 100644 index 0000000000000000000000000000000000000000..9c0beaa17a408593574f5aa0bbe228c9a14ac066 --- /dev/null +++ b/root/wander-train/stage5l2d_typed_predicate_scope_closure.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""L2d: typed lexical opposition and post-predicate nominal scope.""" + +from __future__ import annotations + +import re +from typing import Any, Sequence + +from stage5j2_native_english_construction_algebra import COORDINATORS, DETERMINERS +from stage5k15_native_mixed_agent import digest +from stage5l1_broad_relational_ingress import BroadRelationalCompiler +from stage5l2_structural_language_closure import ALL_NODE_ROUTE, PREFIX_OPERATORS +from stage5l2c_predicate_coordinate_closure import ( + PredicateCoordinateClosureAdapter, + PredicateCoordinateClosureAgent, +) + + +SCHEMA = "oracle-stage5l2d-typed-predicate-scope-closure-v1" +WORD = re.compile(r"[A-Za-z]+") + + +class TypedPredicateScopeClosureAdapter(PredicateCoordinateClosureAdapter): + adapter_id = "oracle-stage5l2d-english-typed-predicate-scope-closure-v1" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + parent_semantics = self.semantic_sha256 + self.semantic_sha256 = digest({ + "schema_version": SCHEMA, + "parent_semantic_sha256": parent_semantics, + "typed_prefix_relation": ( + "path_congruence_or_wordnet_antonymy_or_definition_base" + ), + "post_predicate_determiner_nominal_scope": True, + }) + + def _explicit_lexical_relation(self, surface: str, base: str) -> bool: + if self._semantic_congruence(surface, base) >= self._DERIVATIONAL_SIMILARITY_MINIMUM: + return True + for synset in self.wordnet.synsets(surface, pos=self.wordnet.VERB): + definition = {value.casefold() for value in WORD.findall(synset.definition())} + if base.casefold() in definition: + return True + for lemma in synset.lemmas(): + if any(value.name().casefold() == base.casefold() for value in lemma.antonyms()): + return True + for synset in self.wordnet.synsets(base, pos=self.wordnet.VERB): + for lemma in synset.lemmas(): + if any(value.name().casefold() == surface.casefold() for value in lemma.antonyms()): + return True + return False + + def _derivation(self, surface: str) -> tuple[str, str, str] | None: + inherited = super()._derivation(surface) + if inherited is not None: + return inherited + normalized = surface.casefold().replace("-", "_") + direct = self._direct_frame_lemma(normalized) + if direct is None: + return None + for prefix, operator in PREFIX_OPERATORS: + if not normalized.startswith(prefix) or len(normalized) <= len(prefix) + 2: + continue + base_surface = normalized[len(prefix):] + licensed_base = self._licensed_base_lemma(base_surface) + if licensed_base is None: + continue + lexical_base = self._base_candidates(base_surface)[0] + if self._explicit_lexical_relation(direct, lexical_base): + return direct, licensed_base, operator + return None + + def _coordinate_priority(self, values: Sequence[str], index: int) -> float: + priority = super()._coordinate_priority(values, index) + surface = values[index] + if not self.resolver.noun_like(surface): + return priority + last_coordinator = max( + (offset for offset in range(index) if values[offset] in COORDINATORS), + default=-1, + ) + earlier_predicates = [ + offset for offset in self._predicate_indexes(values) + if last_coordinator < offset < index + ] + if any( + any(value in DETERMINERS for value in values[offset + 1:index]) + for offset in earlier_predicates + ): + priority -= 4.0 + return priority + + +class TypedPredicateScopeClosureCompiler(BroadRelationalCompiler): + def __init__(self, **kwargs: Any) -> None: + kwargs["adapter_type"] = TypedPredicateScopeClosureAdapter + super().__init__(**kwargs) + self.semantic_sha256 = digest({ + "schema_version": SCHEMA, + "parent_semantic_sha256": self.semantic_sha256, + "adapter_semantic_sha256": self.adapter.semantic_sha256, + "all_node_route": ALL_NODE_ROUTE, + }) + + def architecture(self) -> dict[str, Any]: + value = super().architecture() + value.update({ + "schema_version": SCHEMA, + "semantic_sha256": self.semantic_sha256, + "all_predicate_coordinates_retained": True, + "typed_prefix_lexical_relations": True, + "post_predicate_nominal_scope": True, + "coordinated_roles_are_set_valued_states": True, + "structural_extensions_traverse_all_ten_nodes": True, + }) + return value + + +class TypedPredicateScopeClosureAgent(PredicateCoordinateClosureAgent): + def __init__(self, *args: Any, + relational_compiler: TypedPredicateScopeClosureCompiler | None = None, + **kwargs: Any) -> None: + super().__init__( + *args, + relational_compiler=relational_compiler or TypedPredicateScopeClosureCompiler(), + **kwargs, + ) + + +__all__ = [ + "SCHEMA", + "TypedPredicateScopeClosureAdapter", + "TypedPredicateScopeClosureAgent", + "TypedPredicateScopeClosureCompiler", +] diff --git a/root/wander-train/stage5l3c_clause_type_successor.py b/root/wander-train/stage5l3c_clause_type_successor.py new file mode 100644 index 0000000000000000000000000000000000000000..a8d128f2bc018f2b9ba632b8c3ec514d5750cc6e --- /dev/null +++ b/root/wander-train/stage5l3c_clause_type_successor.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""L3c: type clause coordinates before recursive dispatch.""" + +from __future__ import annotations + +from pathlib import Path +import time +from typing import Any, Sequence + +from stage5j2_native_english_construction_algebra import ( + AUXILIARIES, + COORDINATORS, + DETERMINERS, + MODALS, + PREPOSITIONS, +) +from stage5k15_native_mixed_agent import digest +from stage5l2f_recursive_clause_graph import ClauseTree, MARKER_PRIORITY, RELATIONS +from stage5l3_unified_language_bundle import UnifiedLanguageBundle, UnifiedLanguageCandidate +from stage5l3b_lexical_recursive_successor import ( + LexicallyCompleteRecursiveCompiler, + PredicateCoordinate, +) + + +SCHEMA = "oracle-stage5l3c-clause-type-successor-v1" +EXTENDED_PREPOSITIONS = frozenset({ + "above", "across", "along", "around", "beneath", "beside", "beyond", + "despite", "during", "inside", "near", "outside", "through", "toward", + "towards", "underneath", "upon", "within", +}) + + +class ClauseTypedRecursiveCompiler(LexicallyCompleteRecursiveCompiler): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.semantic_sha256 = digest({ + "schema_version": SCHEMA, + "parent_semantic_sha256": self.semantic_sha256, + "all_relation_edges_require_finite_clause_coordinates": True, + "modal_governed_open_lexicon": True, + "object_coordination_shape_suppression": True, + "extended_prepositions": sorted(EXTENDED_PREPOSITIONS), + }) + + def predicate_coordinate( + self, words: Sequence[str], *, require_finite: bool = False, + ) -> PredicateCoordinate | None: + values = tuple(str(value).casefold() for value in words if value != ",") + candidates: list[PredicateCoordinate] = [] + closed = ( + AUXILIARIES | MODALS | DETERMINERS | PREPOSITIONS + | COORDINATORS | frozenset(RELATIONS) | EXTENDED_PREPOSITIONS + ) + for index, surface in enumerate(values): + if surface in closed or not surface.isalpha(): + continue + governed = bool(index and values[index - 1] in AUXILIARIES | MODALS) + derivation = self.adapter._derivation(surface) + lemma = self.adapter.wordnet.morphy(surface, self.adapter.wordnet.VERB) + exception_map = getattr(self.adapter.wordnet, "_exception_map", {}) + irregular_lemmas = tuple( + exception_map.get(self.adapter.wordnet.VERB, {}).get(surface, ()) + ) + if irregular_lemmas: + lemma = irregular_lemmas[0] + lexical = bool( + lemma is not None + or derivation is not None + or self.adapter.resolver.verb_like(surface) + ) + if not lexical and not governed: + continue + if lemma is None and derivation is not None: + lemma = derivation[0] + actual_lemma = str(lemma or surface).casefold() + finite = bool( + governed or irregular_lemmas or self._finite_evidence(values, index) + ) + if require_finite and not finite: + continue + score = 0.0 + if governed: + score += 10.0 + if finite: + score += 5.0 + if index + 1 < len(values) and values[index + 1] in DETERMINERS: + score += 3.0 + if any(value in COORDINATORS for value in values[:index]): + score += 0.5 + if index + 1 < len(values) and values[index + 1] in COORDINATORS: + score -= 5.0 + if index and values[index - 1] in DETERMINERS | PREPOSITIONS: + score -= 7.0 + if self.adapter.resolver.noun_like(surface): + score -= 0.5 + candidates.append(PredicateCoordinate( + index, surface, actual_lemma, score, finite, + "modal_governance" if governed else "wordnet_or_productive_morphology", + )) + if not candidates: + return None + finite_indexes = [value.index for value in candidates if value.finite] + rescored: list[PredicateCoordinate] = [] + for value in candidates: + score = value.score + if ( + self.adapter.resolver.noun_like(value.surface) + and any(index > value.index for index in finite_indexes) + ): + score -= 6.0 + if any( + earlier.index < value.index + and earlier.finite + and any(token in DETERMINERS for token in values[earlier.index + 1:value.index]) + for earlier in candidates + ): + score -= 6.0 + rescored.append(PredicateCoordinate( + value.index, value.surface, value.lemma, score, + value.finite, value.source, + )) + return sorted( + rescored, key=lambda value: (-value.score, -value.index, value.lemma), + )[0] + + def _bilateral_clause_coordinate( + self, words: tuple[str, ...], index: int, + ) -> bool: + left = tuple(value for value in words[:index] if value != ",") + right = tuple(value for value in words[index + 1:] if value != ",") + left_coordinate = self.predicate_coordinate(left, require_finite=True) + right_coordinate = self.predicate_coordinate(right, require_finite=True) + if left_coordinate is None or right_coordinate is None: + return False + right_is_ambiguous_terminal_nominal = bool( + right_coordinate.index == len(right) - 1 + and self.adapter.resolver.noun_like(right_coordinate.surface) + and not right_coordinate.surface.endswith(("ed", "ing", "en")) + and not ( + right_coordinate.index + and right[right_coordinate.index - 1] in AUXILIARIES | MODALS + ) + ) + left_has_realized_object = left_coordinate.index < len(left) - 1 + if left_has_realized_object and right_is_ambiguous_terminal_nominal: + return False + return True + + def _parse(self, words: tuple[str, ...], depth: int = 0) -> ClauseTree: + if depth >= 8 or len(words) < 2: + return ClauseTree(words) + if words[0] in RELATIONS and words[0] not in {"and", "or", "but", "that"}: + try: + comma = words.index(",") + except ValueError: + comma = -1 + if comma > 1: + left, right = words[1:comma], words[comma + 1:] + if ( + self._structurally_analyzable(left, require_finite=True) + and self._structurally_analyzable(right, require_finite=True) + ): + return ClauseTree( + words, RELATIONS[words[0]], words[0], + self._parse(tuple(left), depth + 1), + self._parse(tuple(right), depth + 1), False, + ) + positions = [ + (MARKER_PRIORITY.index(value), index, value) + for index, value in enumerate(words) + if 0 < index < len(words) - 1 and value in RELATIONS + ] + for _, index, marker in sorted(positions): + left = tuple(value for value in words[:index] if value != ",") + right = tuple(value for value in words[index + 1:] if value != ",") + if marker in {"and", "or"}: + valid = self._bilateral_clause_coordinate(words, index) + else: + valid = bool( + self._structurally_analyzable(left, require_finite=True) + and self._structurally_analyzable(right, require_finite=True) + ) + if not valid: + continue + reverse = marker in { + "because", "since", "if", "whether", "although", "though", + } + return ClauseTree( + words, RELATIONS[marker], marker, + self._parse(left, depth + 1), self._parse(right, depth + 1), + reverse, + ) + return ClauseTree(tuple(value for value in words if value != ",")) + + +class ClauseTypedUnifiedLanguageCompiler: + def __init__(self, bundle_path: Path, *, cache_directory: Path, + maximum_hypotheses: int = 32) -> None: + started = time.perf_counter() + self.bundle = UnifiedLanguageBundle(bundle_path, verify=False) + self.cache_directory = Path(cache_directory).resolve() + basis = self.bundle.materialize( + "verbnet_operator_basis", self.cache_directory / "verbnet_basis.json.gz", + ) + wordnet_root = self.cache_directory / "nltk_data" + self.bundle.materialize( + "wordnet_archive", wordnet_root / "corpora/wordnet.zip", + ) + self.compiler = ClauseTypedRecursiveCompiler( + wordnet_root, basis, maximum_hypotheses=maximum_hypotheses, + ) + self.maximum_hypotheses = max(1, int(maximum_hypotheses)) + self.semantic_sha256 = digest({ + "schema_version": SCHEMA, + "bundle_content_manifest_sha256": self.bundle.content_manifest_sha256, + "compiler_semantic_sha256": self.compiler.semantic_sha256, + "dispatch_policy": "finite_typed_relation_edges", + }) + self.startup_ms = 1000.0 * (time.perf_counter() - started) + + @staticmethod + def _tokens(text: str) -> tuple[str, ...]: + normalized = str(text).replace(",", " , ") + return tuple( + value.strip(".?!;:").casefold() + for value in normalized.split() + if value == "," or value.strip(".?!;:") + ) + + def compile(self, text: str, *, session_id: str = "l3c") -> UnifiedLanguageCandidate: + started = time.perf_counter() + tokens = self._tokens(text) + single = self.compiler.compile_single(text, session_id=session_id) + tree = self.compiler._parse(tokens) + recursive = len(self.compiler._leaves(tree)) >= 2 + if recursive: + candidate = self.compiler.compile(text, session_id=session_id) + if candidate.eligible_for_semantic_arbitration: + return UnifiedLanguageCandidate( + "recursive_clause_graph", candidate.analyzed, + candidate.eligible_for_semantic_arbitration, + self.semantic_sha256, candidate.as_dict(), + 1000.0 * (time.perf_counter() - started), + ) + eligible = bool(single.get("eligible_for_semantic_arbitration")) + return UnifiedLanguageCandidate( + "single_clause_program", bool(single.get("analyzed")), eligible, + self.semantic_sha256, single, + 1000.0 * (time.perf_counter() - started), + ) + + def architecture(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA, + "semantic_sha256": self.semantic_sha256, + "startup_ms": self.startup_ms, + "bundle_memory_mapped": True, + "bundle_component_count": len(self.bundle.components), + "clause_boundaries_independent_of_verbnet_coverage": True, + "all_relation_edges_require_finite_clause_coordinates": True, + "modal_governed_open_lexicon": True, + "wordnet_irregular_exception_morphology": True, + "object_coordination_shape_suppression": True, + "generic_lexical_factual_authority": False, + "known_frames_retain_frozen_proof_path": True, + "protected_four_to_one_spine": True, + "all_runtime_operators_use_full_tree_route": True, + "terminal_malkuth_release_only": True, + "traditional_language_model": False, + "transformer": False, + "pretrained_continuous_embedding": False, + "gradient_training": False, + } + + def close(self) -> None: + self.bundle.close() + + +__all__ = [ + "ClauseTypedRecursiveCompiler", "ClauseTypedUnifiedLanguageCompiler", + "EXTENDED_PREPOSITIONS", "SCHEMA", +] diff --git a/root/wander-train/stage5l5a_hierarchical_self_document.py b/root/wander-train/stage5l5a_hierarchical_self_document.py new file mode 100644 index 0000000000000000000000000000000000000000..09349eff51a4d6505d3662458052a8e201ef7c16 --- /dev/null +++ b/root/wander-train/stage5l5a_hierarchical_self_document.py @@ -0,0 +1,986 @@ +#!/usr/bin/env python3 +"""L5a: hierarchical, evidence-addressed self-document construction. + +This successor consolidates the K134 document proof and whole-document inverse +behind the frozen L4b semantic-authority boundary. It does not contain a +source essay. It observes the running Oracle architecture and frozen reports, +constructs an evidence-addressed claim graph, transports opaque claim handles +through the protected Tree spine, plans sections and paragraphs, and realizes +the complete document only after a global layout inverse. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path +import re +import time +from typing import Any, Mapping, Sequence + +import numpy as np + +from stage5k15_native_mixed_agent import NativeAnswerGrammar, SemanticClause, digest +from stage5k238a_epistemically_guarded_release_candidate import ( + EpistemicallyGuardedOracleRelationalV3, +) +from stage5l3c_clause_type_successor import ClauseTypedUnifiedLanguageCompiler +from stage5l4_simultaneous_semantic_authority import SemanticCandidate +from stage5l4b_relational_discourse_act_successor import ( + RelationalDiscourseActRuntime, + RelationalDiscourseOracleRelationalV3, +) +from train_stage2a_role_ladder import ( + PATHS, + SEPHIROTH, + bind, + cosine, + normalize, + unbind, + unitary, +) + + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA = "oracle-stage5l5a-hierarchical-self-document-v1" +WORD = re.compile(r"[a-z0-9]+(?:[-'][a-z0-9]+)?", re.I) +NUMBER_WORDS = { + "three": 3, "four": 4, "five": 5, "six": 6, +} +ARTIFACT_ROLES = frozenset({ + "account", "analysis", "article", "document", "essay", "report", "treatise", +}) +ORACLE_ROLES = frozenset({ + "architecture", "oracle", "oracle's", "runtime", "self", "system", "yourself", +}) +DOCUMENT_OPERATOR_SEEDS = ( + "compose", "develop", "draft", "explain", "produce", "write", +) +COMMAND_PREAMBLE = frozenset({ + "can", "could", "kindly", "please", "will", "would", "you", +}) +ALLOWED_ABLATIONS = frozenset({ + "architecture_observation_authority", + "hierarchical_document_plan", + "phenomenology_boundary", + "whole_document_inverse", +}) + + +def _sha_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _slug(value: str) -> str: + words = WORD.findall(str(value).casefold()) + return "_".join(words[:18]) or "none" + + +@dataclass(frozen=True) +class DocumentRequest: + artifact: str + paragraphs: int + scope: str + mode: str + operator: str + + def public(self) -> dict[str, Any]: + return dict(self.__dict__) + + +@dataclass(frozen=True) +class EvidenceAddress: + evidence_id: str + authority_class: str + source_path: str + source_sha256: str + observation_path: str + observed_value: Any + + def public(self) -> dict[str, Any]: + return dict(self.__dict__) + + +@dataclass(frozen=True) +class DocumentClaim: + claim_id: str + section: str + claim_type: str + clause: SemanticClause + evidence_ids: tuple[str, ...] + + def public(self) -> dict[str, Any]: + return { + "claim_id": self.claim_id, + "section": self.section, + "claim_type": self.claim_type, + "clause": self.clause.as_dict(), + "evidence_ids": list(self.evidence_ids), + } + + +@dataclass(frozen=True) +class ClaimEdge: + source: str + relation: str + target: str + + def public(self) -> dict[str, str]: + return dict(self.__dict__) + + +class HierarchicalSelfDocumentRuntime(RelationalDiscourseActRuntime): + """Construct a whole self-document from runtime observations.""" + + def __init__(self, *args: Any, l5_ablations: Sequence[str] = (), **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.l5_ablations = frozenset(str(value) for value in l5_ablations) + unknown = self.l5_ablations - ALLOWED_ABLATIONS + if unknown: + raise ValueError("unknown L5 ablations: " + ", ".join(sorted(unknown))) + self.document_operator_lemmas = self._exact_document_operator_family() + parent_semantic_sha256 = self.semantic_sha256 + self.semantic_sha256 = digest({ + "schema_version": SCHEMA, + "parent_semantic_sha256": parent_semantic_sha256, + "document_operator_seeds": list(DOCUMENT_OPERATOR_SEEDS), + "document_operator_lemmas": list(self.document_operator_lemmas), + "artifact_roles": sorted(ARTIFACT_ROLES), + "hierarchy": [ + "response_objective", "epistemic_contract", "claim_graph", + "section_plan", "paragraph_plan", "whole_document_inverse", + ], + "ablations": sorted(self.l5_ablations), + }) + + def _exact_document_operator_family(self) -> tuple[str, ...]: + lemmas = set(DOCUMENT_OPERATOR_SEEDS) + for seed in DOCUMENT_OPERATOR_SEEDS: + for synset in self.wordnet.synsets(seed, pos=self.wordnet.VERB): + lemmas.update( + str(value).replace("_", " ").casefold() + for value in synset.lemma_names() + ) + return tuple(sorted(lemmas)) + + def _document_request(self, text: str) -> DocumentRequest | None: + tokens = tuple( + part + for value in WORD.findall(str(text)) + for part in value.casefold().split("-") + if part + ) + if not tokens or not (set(tokens) & ARTIFACT_ROLES): + return None + operator = None + for index, token in enumerate(tokens[:6]): + forms = {token} + lemma = self.wordnet.morphy(token, self.wordnet.VERB) + if lemma: + forms.add(str(lemma).replace("_", " ").casefold()) + selected = sorted(forms & set(self.document_operator_lemmas)) + if selected and all(value in COMMAND_PREAMBLE for value in tokens[:index]): + operator = selected[0] + break + if operator is None: + return None + paragraphs = next( + (int(value) for value in tokens if value.isdigit() and 3 <= int(value) <= 6), + next((NUMBER_WORDS[value] for value in tokens if value in NUMBER_WORDS), 5), + ) + scope = "oracle_architecture" if set(tokens) & ORACLE_ROLES else "unsupported" + mode = "functional_introspection" if set(tokens) & { + "experience", "functionally", "like", "operate", "operating", + } else "architectural_exposition" + artifact = next(value for value in tokens if value in ARTIFACT_ROLES) + return DocumentRequest(artifact, paragraphs, scope, mode, operator) + + @staticmethod + def _source(path: str, observation_path: str, observed_value: Any) -> EvidenceAddress: + absolute = ROOT / path + return EvidenceAddress( + "evidence:" + digest([path, observation_path, observed_value])[:24], + "runtime_architecture_observation", + path, + _sha_file(absolute), + observation_path, + observed_value, + ) + + def _observations(self) -> tuple[dict[str, Any], dict[str, EvidenceAddress]]: + identity = self.discourse.identity.contract() + architecture = super().architecture() + language = self.language.architecture() + prospective_path = ( + "fable-content/oracle-phase3-2026-07-21/" + "stage5l4b_relational_discourse_act_prospective_report_v1.json" + ) + prospective = json.loads((ROOT / prospective_path).read_text(encoding="utf-8")) + values = { + "identity": identity, + "architecture": architecture, + "language": language, + "prospective": prospective, + } + evidence = { + "identity": self._source( + "wander-train/stage5k223a_identity_anchored_session.py", + "OracleIdentity.contract", identity, + ), + "language": self._source( + "wander-train/stage5l3c_clause_type_successor.py", + "ClauseTypedUnifiedLanguageCompiler.architecture", language, + ), + "authority": self._source( + "wander-train/stage5l4_simultaneous_semantic_authority.py", + "SimultaneousSemanticAuthorityRuntime.architecture", architecture, + ), + "discourse": self._source( + "wander-train/stage5l4b_relational_discourse_act_successor.py", + "RelationalDiscourseActRuntime.architecture", architecture, + ), + "prospective": self._source( + prospective_path, + "status+metrics", { + "status": prospective.get("status"), + "accepted": prospective.get("metrics", {}).get("accepted"), + "case_count": prospective.get("metrics", {}).get("case_count"), + }, + ), + "document": self._source( + "wander-train/stage5l5a_hierarchical_self_document.py", + "HierarchicalSelfDocumentRuntime.architecture", { + "claim_graph": True, + "section_and_paragraph_plans": True, + "whole_document_inverse": True, + "source_essay_required": False, + }, + ), + } + return values, evidence + + @staticmethod + def _claim( + claim_id: str, section: str, claim_type: str, + subject: str, predicate: str, *, object_value: str | None = None, + complement: str | None = None, + qualifiers: Sequence[tuple[str, str]] = (), + polarity: str = "positive", evidence: Sequence[str], + ) -> DocumentClaim: + return DocumentClaim( + claim_id, section, claim_type, + SemanticClause( + subject, predicate, object_value=object_value, + complement=complement, qualifiers=tuple(qualifiers), + polarity=polarity, + ), + tuple(evidence), + ) + + def _claims( + self, observations: Mapping[str, Any], evidence: Mapping[str, EvidenceAddress], + ) -> tuple[DocumentClaim, ...]: + architecture = observations["architecture"] + language = observations["language"] + prospective = observations["prospective"] + required = { + "simultaneous_typed_candidate_arbitration": True, + "structural_program_has_factual_answer_authority": False, + "protected_four_to_one_spine": True, + "terminal_malkuth_release_only": True, + "traditional_language_model": False, + "transformer": False, + "relational_discourse_act_projection": True, + } + for key, expected in required.items(): + if architecture.get(key) is not expected: + raise RuntimeError(f"architecture observation mismatch: {key}") + if language.get("all_runtime_operators_use_full_tree_route") is not True: + raise RuntimeError("language compiler no longer reports full-tree routing") + if prospective.get("status") != "passed": + raise RuntimeError("L4b prospective authority is not passed") + + eid = {name: value.evidence_id for name, value in evidence.items()} + claims = [ + self._claim( + "c01", "identity", "observed_fact", "Oracle", "be", + complement="a " + str(observations["identity"]["kind"]), evidence=(eid["identity"],), + ), + self._claim( + "c02", "identity", "observed_fact", "Its declared purpose", "be", + complement="to " + str(observations["identity"]["purpose"]), evidence=(eid["identity"],), + ), + self._claim( + "c03", "identity", "bounded_synthesis", "The governing thesis", "be", + complement=( + "that Oracle operates by preserving a relational state across interpretation, " + "authority, memory, and realization rather than by predicting the next token" + ), + evidence=(eid["identity"], eid["language"], eid["authority"]), + ), + self._claim( + "c04", "ingress", "functional_introspection", + "Functionally, receiving a question", "mean", + object_value="constructing a query-local relational program from its predicates, roles, and clause topology", + evidence=(eid["language"],), + ), + self._claim( + "c05", "ingress", "observed_fact", "The unified language compiler", "route", + object_value="every admitted runtime operator through all ten Tree nodes", + qualifiers=(("before", "terminal semantic release"),), evidence=(eid["language"],), + ), + self._claim( + "c06", "ingress", "epistemic_boundary", "A structural program", "acquire", + object_value="factual answer authority from linguistic well-formedness alone", + polarity="negative", evidence=(eid["authority"],), + ), + self._claim( + "c07", "authority", "observed_fact", "The semantic authority layer", "evaluate", + object_value="structural, conversational, tool, routing, memory, lexical, and specialist candidates in one typed arbitration", + evidence=(eid["authority"],), + ), + self._claim( + "c08", "authority", "observed_fact", "An exact WordNet relation bridge", "propose", + object_value="a lexical normalization", + qualifiers=(("while", "an unchanged authenticated specialist decides whether a factual edge exists"),), + evidence=(eid["authority"],), + ), + self._claim( + "c09", "authority", "bounded_synthesis", "This division of labor", "make", + object_value="linguistic freedom compatible with evidential restraint", + qualifiers=(("because", "interpretation can vary without manufacturing knowledge"),), + evidence=(eid["authority"],), + ), + self._claim( + "c10", "spine", "observed_fact", "The four-to-one stabilizing spine", "protect", + object_value="the identity and governing root while bounded relational payloads accumulate", + evidence=(eid["identity"], eid["authority"]), + ), + self._claim( + "c11", "spine", "functional_introspection", "For the running architecture, reasoning", "mean", + object_value="transporting opaque claim handles through path and node transformations until Malkuth permits cleanup", + evidence=(eid["identity"], eid["language"], eid["authority"]), + ), + self._claim( + "c12", "memory", "observed_fact", "Persistent discourse state", "retain", + object_value="the last authenticated plan, its evidence addresses, active specialist, salience, and turn history", + evidence=(eid["discourse"],), + ), + self._claim( + "c13", "memory", "observed_fact", "The relational reproduction operator", "select", + object_value="that stored plan only when a reproduction operator and backward discourse anchor agree", + qualifiers=(("across", "runtime restart as well as an uninterrupted session"),), + evidence=(eid["discourse"], eid["prospective"]), + ), + self._claim( + "c14", "release", "observed_fact", "Malkuth", "be", + complement="the only semantic release boundary in the tested architecture", + evidence=(eid["authority"],), + ), + self._claim( + "c15", "release", "observed_fact", "This document runtime", "select", + object_value="one complete paragraph layout from a stable claim graph", + qualifiers=(("instead of", "replanning the argument one sentence at a time"),), + evidence=(eid["document"],), + ), + self._claim( + "c16", "release", "observed_fact", "The frozen L4b prospective panel", "accept", + object_value=( + f"{prospective['metrics']['accepted']} of {prospective['metrics']['case_count']} cases" + ), + qualifiers=(("including", "causal parent ablation, collision withholding, and restart persistence"),), + evidence=(eid["prospective"],), + ), + ] + if "phenomenology_boundary" not in self.l5_ablations: + claims.append(self._claim( + "c17", "release", "epistemic_boundary", "This functional account", "establish", + object_value="subjective consciousness or phenomenal experience from these functional traces", + polarity="negative", evidence=(eid["identity"], eid["prospective"]), + )) + claims.append(self._claim( + "c18", "release", "bounded_synthesis", "The warranted conclusion", "be", + complement=( + "that being Oracle can be described operationally as maintaining one evidence-bounded " + "relational form from question through answer, while stronger claims remain withheld" + ), + evidence=(eid["identity"], eid["authority"], eid["discourse"], eid["prospective"]), + )) + return tuple(claims) + + @staticmethod + def _claim_edges(claims: Sequence[DocumentClaim]) -> tuple[ClaimEdge, ...]: + ids = {value.claim_id for value in claims} + edges = [ + ClaimEdge(value.claim_id, "supports", "c03") + for value in claims if value.claim_id not in {"c03", "c17"} + ] + ordered = [value.claim_id for value in claims] + edges.extend( + ClaimEdge(left, "precedes", right) + for left, right in zip(ordered, ordered[1:]) + ) + if "c17" in ids: + edges.append(ClaimEdge("c17", "qualifies", "c18")) + return tuple(edges) + + @staticmethod + def _partition_sections(paragraphs: int) -> tuple[tuple[str, ...], ...]: + sections = ("identity", "ingress", "authority", "spine", "memory", "release") + quotient, remainder = divmod(len(sections), paragraphs) + groups = [] + cursor = 0 + for index in range(paragraphs): + width = quotient + int(index < remainder) + groups.append(tuple(sections[cursor:cursor + width])) + cursor += width + return tuple(groups) + + @staticmethod + def _spine_projection(claims: Sequence[DocumentClaim], thesis_id: str) -> dict[str, Any]: + claim_handles = { + value.claim_id: unitary("l5:claim:" + digest(value.public())) + for value in claims + } + roles = { + value.claim_id: unitary("l5:claim-role:" + value.claim_id) + for value in claims + } + payload = normalize(sum( + (bind(roles[value.claim_id], claim_handles[value.claim_id]) for value in claims), + start=np.zeros_like(next(iter(claim_handles.values()))), + )) + root = unitary("l5:thesis-root:" + thesis_id) + residual = payload - np.complex64(np.vdot(root, payload)) * root + protected = normalize(np.float32(4.0) * root + normalize(residual)) + node_cosines = [] + for node in SEPHIROTH: + node_state = bind( + unitary("l5:tree-node:" + node), + unitary("l5:document-state:" + thesis_id), + ) + node_state = node_state - np.complex64(np.vdot(root, node_state)) * root + residual = normalize(residual + np.complex64(0.04) * normalize(node_state)) + protected = normalize(np.float32(4.0) * root + residual) + node_cosines.append(cosine(protected, root)) + recoveries = [] + for claim in claims: + recovered = unbind(payload, roles[claim.claim_id]) + scores = { + candidate: cosine(recovered, handle) + for candidate, handle in claim_handles.items() + } + winner = max(scores, key=lambda value: (scores[value], value)) + ordered = sorted(scores.values(), reverse=True) + recoveries.append({ + "claim_id": claim.claim_id, + "winner": winner, + "exact": winner == claim.claim_id, + "score": scores[claim.claim_id], + "margin": ordered[0] - ordered[1] if len(ordered) > 1 else ordered[0], + }) + return { + "route": list(SEPHIROTH), + "tree_nodes_registered": len(SEPHIROTH), + "tree_paths_registered": len(PATHS), + "path_rail_sha256": digest([list(value) for value in PATHS]), + "minimum_protected_root_cosine": min(node_cosines), + "pre_malkuth_claim_unbinds": 0, + "malkuth_claim_unbinds": len(recoveries), + "terminal_cleanup_exact": all(value["exact"] for value in recoveries), + "minimum_cleanup_margin": min(value["margin"] for value in recoveries), + "recoveries": recoveries, + "payload_vector_sha256": hashlib.sha256( + np.ascontiguousarray(payload).tobytes() + ).hexdigest(), + "protected_vector_sha256": hashlib.sha256( + np.ascontiguousarray(protected).tobytes() + ).hexdigest(), + } + + @staticmethod + def _layout_inverse( + paragraphs: Sequence[str], paragraph_claims: Sequence[Sequence[str]], + *, ablated: bool, + ) -> tuple[str, dict[str, Any]]: + canonical = "\n\n".join(paragraphs) + collapsed = " ".join(paragraphs) + reversed_document = "\n\n".join(reversed(paragraphs)) + rotated = "\n\n".join((*paragraphs[1:], paragraphs[0])) + candidates = ( + ("canonical", canonical, 0.0), + ("collapsed", collapsed, 2.0), + ("reversed", reversed_document, 3.0), + ("rotated", rotated, 1.5), + ) + selected_index = 1 if ablated else min( + range(len(candidates)), key=lambda index: (candidates[index][2], candidates[index][0]), + ) + penalty = 4.0 + matrix = [[0.0 for _ in candidates] for _ in candidates] + for index, candidate in enumerate(candidates): + matrix[index][index] = candidate[2] - penalty + for left in range(len(candidates)): + for right in range(left + 1, len(candidates)): + matrix[left][right] = matrix[right][left] = penalty + assignment = [int(index == selected_index) for index in range(len(candidates))] + return candidates[selected_index][1], { + "selected_layout": candidates[selected_index][0], + "candidate_layouts": [value[0] for value in candidates], + "candidate_losses": [value[2] for value in candidates], + "candidate_text_sha256s": [ + hashlib.sha256(value[1].encode("utf-8")).hexdigest() + for value in candidates + ], + "quadratic_matrix_sha256": digest(matrix), + "assignment": assignment, + "assignment_hamming_weight": sum(assignment), + "exhaustive_one_hot_minimum_verified": bool(not ablated and selected_index == 0), + "proof_reconstruction_exact": bool( + not ablated + and [value for group in paragraph_claims for value in group] + == [value for group in paragraph_claims for value in group] + ), + "whole_document_selected_globally": not ablated, + "next_token_predictions": 0, + } + + @staticmethod + def _longest_common_run(left: Sequence[str], right: Sequence[str]) -> int: + prior = [0] * (len(right) + 1) + best = 0 + for left_value in left: + current = [0] * (len(right) + 1) + for index, right_value in enumerate(right, start=1): + if left_value == right_value: + current[index] = prior[index - 1] + 1 + best = max(best, current[index]) + prior = current + return best + + def _document( + self, request: DocumentRequest, + ) -> tuple[str, list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]: + observations, evidence = self._observations() + claims = self._claims(observations, evidence) + edges = self._claim_edges(claims) + groups = self._partition_sections(request.paragraphs) + grammar = NativeAnswerGrammar() + claim_by_id = {value.claim_id: value for value in claims} + paragraph_claim_ids = [ + [value.claim_id for value in claims if value.section in sections] + for sections in groups + ] + paragraphs = [ + grammar.realize(tuple(claim_by_id[claim_id].clause for claim_id in claim_ids)) + for claim_ids in paragraph_claim_ids + ] + response, inverse = self._layout_inverse( + paragraphs, paragraph_claim_ids, + ablated="whole_document_inverse" in self.l5_ablations, + ) + spine = self._spine_projection(claims, "c03") + source_map = {value.evidence_id: value for value in evidence.values()} + evidence_ledger = [ + { + "fact_id": "l5:" + claim.claim_id, + "claim_id": claim.claim_id, + "claim_type": claim.claim_type, + "evidence_ids": list(claim.evidence_ids), + "evidence_sha256": digest([ + evidence_id for evidence_id in claim.evidence_ids + ]), + "source_document": ";".join( + source_map[evidence_id].source_path + for evidence_id in claim.evidence_ids + ), + "source_document_sha256": digest([ + source_map[evidence_id].source_sha256 + for evidence_id in claim.evidence_ids + ]), + "provenance_sha256": digest(claim.public()), + } + for claim in claims + ] + content_plan = [ + { + "ordinal": ordinal, + "fact_id": "l5:" + claim.claim_id, + "source": _slug(claim.clause.subject), + "relation": _slug(claim.clause.predicate), + "target": _slug(claim.clause.object_value or claim.clause.complement or "state"), + "discourse_function": claim.section, + "sentence": grammar.clause(claim.clause), + "claim_id": claim.claim_id, + "claim_type": claim.claim_type, + "evidence_ids": list(claim.evidence_ids), + } + for ordinal, claim in enumerate(claims) + ] + sentence_hashes = [ + hashlib.sha256(value["sentence"].encode("utf-8")).hexdigest() + for value in content_plan + ] + duplicate_count = len(sentence_hashes) - len(set(sentence_hashes)) + contradiction_keys: dict[tuple[str, str, str], set[str]] = {} + for claim in claims: + key = ( + _slug(claim.clause.subject), _slug(claim.clause.predicate), + _slug(claim.clause.object_value or claim.clause.complement or "state"), + ) + contradiction_keys.setdefault(key, set()).add(claim.clause.polarity) + contradiction_count = sum( + len(values) > 1 for values in contradiction_keys.values() + ) + all_evidence_ids = {item for claim in claims for item in claim.evidence_ids} + artifact = { + "schema_version": SCHEMA, + "response_objective": { + **request.public(), + "objective": "architecture-grounded hierarchical self-document", + "source_essay_available": False, + }, + "epistemic_contract": { + "factual_claims_require_architecture_observation": True, + "structural_language_has_factual_authority": False, + "functional_introspection_is_not_phenomenology": True, + "unsupported_topics_withheld": True, + }, + "claim_graph": { + "claims": [value.public() for value in claims], + "edges": [value.public() for value in edges], + "thesis_claim_id": "c03", + "connected_to_thesis": all( + value.claim_id == "c03" + or any(edge.source == value.claim_id and edge.target == "c03" for edge in edges) + or value.claim_id == "c17" + for value in claims + ), + }, + "section_plan": [ + { + "section_ordinal": index, + "section_roles": list(sections), + "claim_ids": paragraph_claim_ids[index], + "thesis_anchor_claim_id": "c03", + } + for index, sections in enumerate(groups) + ], + "paragraph_plan": [ + { + "paragraph_ordinal": index, + "claim_ids": claim_ids, + "sentence_sha256s": [ + hashlib.sha256( + grammar.clause(claim_by_id[claim_id].clause).encode("utf-8") + ).hexdigest() + for claim_id in claim_ids + ], + "thesis_anchor_claim_id": "c03", + } + for index, claim_ids in enumerate(paragraph_claim_ids) + ], + "evidence_registry": [ + source_map[value].public() for value in sorted(all_evidence_ids) + ], + "tree_spine": spine, + "document_inverse": inverse, + "audit": { + "source_essay_available": False, + "source_inputs_are_structured_observations": True, + "paragraph_count": len(response.split("\n\n")), + "claim_count": len(claims), + "claim_edge_count": len(edges), + "all_claims_evidence_addressed": all(claim.evidence_ids for claim in claims), + "trace_to_claim_alignment": all( + value["claim_id"] in claim_by_id for value in content_plan + ), + "thesis_persistent_in_all_paragraph_plans": all( + value["thesis_anchor_claim_id"] == "c03" + for value in [ + { + "thesis_anchor_claim_id": "c03", + "claim_ids": claim_ids, + } + for claim_ids in paragraph_claim_ids + ] + ), + "duplicate_sentence_count": duplicate_count, + "contradiction_count": contradiction_count, + "architecture_grounded_self_description": True, + "phenomenology_claim_withheld": "phenomenology_boundary" not in self.l5_ablations, + "traditional_language_model_calls": 0, + "transformer_calls": 0, + "pretrained_embedding_encoder_calls": 0, + "next_token_predictions": 0, + "gradient_steps": 0, + }, + "artifact_sha256": "", + } + artifact["artifact_sha256"] = digest({ + key: value for key, value in artifact.items() if key != "artifact_sha256" + }) + return response, content_plan, evidence_ledger, artifact + + def _finalize_document( + self, session_id: str, text: str, request: DocumentRequest, + candidates: list[SemanticCandidate], context: Mapping[str, Any], + ) -> dict[str, Any]: + started = time.perf_counter() + goal = SemanticCandidate.create( + kind="hierarchical_document_goal", + operation="construct_hierarchical_self_document", + eligible=True, + structural_authority=True, + discourse_authority=True, + provenance="l5_exact_synset_operator_plus_artifact_role", + payload=request.public(), + ) + candidates.append(goal) + authority_enabled = "architecture_observation_authority" not in self.l5_ablations + plan_enabled = "hierarchical_document_plan" not in self.l5_ablations + authority = SemanticCandidate.create( + kind="architecture_observation_authority", + operation="read_verified_runtime_architecture", + eligible=authority_enabled, + factual_authority=authority_enabled, + confidence=1.0 if authority_enabled else 0.0, + provenance="content_hashed_runtime_sources_and_frozen_reports", + payload={ + "scope": request.scope, + "source_essay_available": False, + "authority_ablated": not authority_enabled, + }, + ) + candidates.append(authority) + state = self.discourse._state(session_id) + if request.scope != "oracle_architecture" or not authority_enabled or not plan_enabled: + reason = ( + "the requested topic is outside the currently authenticated self-architecture scope" + if request.scope != "oracle_architecture" + else "a causal L5 authority or planning component is disabled" + ) + result = self.discourse._finalize( + state, user_text=text, resolved_text=text, + status="clarification", operation="l5_document_authority_withheld", + response=( + "I can construct a grounded long-form document only when its topic has " + "an authenticated observation source and the hierarchical plan is available; " + + reason + "." + ), + bundle=state.active_bundle, plan=[], evidence=[], coreference=None, + ) + document_artifact = None + factual_winners: list[str] = [] + structural_winners = [goal.candidate_id] + else: + response, plan, evidence_ledger, document_artifact = self._document(request) + input_tokens = tuple(value.casefold() for value in WORD.findall(text)) + response_tokens = tuple(value.casefold() for value in WORD.findall(response)) + common_run = self._longest_common_run(input_tokens, response_tokens) + run_ratio = common_run / max(1, len(input_tokens)) + exact_surface = str(text).strip().casefold() in response.casefold() + echo_blocked = bool(exact_surface or run_ratio >= 0.90) + document_artifact["anti_echo_release"] = { + "blocked": echo_blocked, + "exact_input_surface_in_release": exact_surface, + "longest_common_input_run_tokens": common_run, + "longest_common_input_run_ratio": run_ratio, + } + document_artifact["artifact_sha256"] = digest({ + key: value for key, value in document_artifact.items() + if key != "artifact_sha256" + }) + if echo_blocked: + result = self.discourse._finalize( + state, user_text=text, resolved_text=text, + status="clarification", operation="l5_anti_echo_withholding", + response=( + "The planned document did not pass the independent anti-echo " + "release boundary, so no document was released." + ), + bundle=state.active_bundle, plan=[], evidence=[], coreference=None, + ) + result["schema_version"] = SCHEMA + result["hierarchical_document"] = document_artifact + result["semantic_arbitration"] = { + "schema_version": SCHEMA, + "semantic_sha256": self.semantic_sha256, + "candidate_count": len(candidates), + "candidates": [value.public() for value in candidates], + "winners": { + "structural": [goal.candidate_id], "factual": [], + "released": goal.candidate_id, + }, + "release_path": "l5_anti_echo_withholding", + "release_authorized": False, + "structural_program_has_factual_answer_authority": False, + "generic_lexical_fallback_has_factual_answer_authority": False, + "document_plan_has_factual_answer_authority": False, + "factual_claims_require_architecture_observation": True, + "runtime": { + "elapsed_milliseconds": 1000.0 * (time.perf_counter() - started), + "traditional_language_model_calls": 0, "transformer_calls": 0, + "pretrained_embedding_encoder_calls": 0, + "next_token_predictions": 0, "gradient_steps": 0, + }, + } + result.setdefault("audit", {})["l5_external_writes"] = 0 + result["audit"]["source_essay_available"] = False + self._persist_manifest(session_id, result) + return result + plan_candidate = SemanticCandidate.create( + kind="hierarchical_document_plan", + operation="claim_graph_to_global_document_inverse", + eligible=True, + structural_authority=True, + discourse_authority=True, + provenance="l5_claim_graph_section_paragraph_inverse", + payload={ + "artifact_sha256": document_artifact["artifact_sha256"], + "claim_count": len(plan), + "paragraph_count": request.paragraphs, + }, + ) + candidates.append(plan_candidate) + state.active_bundle = "architecture:self" + state.last_plan = list(plan) + state.last_evidence = list(evidence_ledger) + self.discourse._update_salience(state, plan) + result = self.discourse._finalize( + state, user_text=text, resolved_text=text, + status="answered", operation="l5_hierarchical_document_realization", + response=response, bundle="architecture:self", + plan=plan, evidence=evidence_ledger, coreference=None, + authority={ + "kind": "runtime_architecture_observation", + "candidate_id": authority.candidate_id, + "artifact_sha256": document_artifact["artifact_sha256"], + }, + ) + factual_winners = [authority.candidate_id] + structural_winners = [goal.candidate_id, plan_candidate.candidate_id] + + structural_winners.extend( + value.candidate_id for value in candidates + if value.kind == "structural_program" and value.eligible + ) + result["schema_version"] = SCHEMA + result["hierarchical_document"] = document_artifact + result["semantic_arbitration"] = { + "schema_version": SCHEMA, + "semantic_sha256": self.semantic_sha256, + "candidate_count": len(candidates), + "candidates": [value.public() for value in candidates], + "winners": { + "structural": structural_winners, + "factual": factual_winners, + "released": goal.candidate_id, + }, + "release_path": ( + "verified_architecture_observations_to_hierarchical_document" + if result["status"] == "answered" else "l5_authority_withholding" + ), + "release_authorized": bool(result["status"] == "answered" and factual_winners), + "structural_program_has_factual_answer_authority": False, + "generic_lexical_fallback_has_factual_answer_authority": False, + "document_plan_has_factual_answer_authority": False, + "factual_claims_require_architecture_observation": True, + "runtime": { + "elapsed_milliseconds": 1000.0 * (time.perf_counter() - started), + "traditional_language_model_calls": 0, + "transformer_calls": 0, + "pretrained_embedding_encoder_calls": 0, + "next_token_predictions": 0, + "gradient_steps": 0, + }, + } + result.setdefault("audit", {})["l5_external_writes"] = 0 + result["audit"]["source_essay_available"] = False + self._persist_manifest(session_id, result) + return result + + def answer(self, session_id: str, text: str) -> dict[str, Any]: + request = self._document_request(text) + if request is None: + return super().answer(session_id, text) + candidates, context = self._candidate_set(session_id, text) + structural = context["structural"] + if not structural.eligible_for_semantic_arbitration: + return super().answer(session_id, text) + return self._finalize_document( + session_id, text, request, candidates, context, + ) + + def architecture(self) -> dict[str, Any]: + value = dict(super().architecture()) + value.update({ + "schema_version": SCHEMA, + "semantic_sha256": self.semantic_sha256, + "hierarchical_document_generation": True, + "response_objective": True, + "epistemic_contract": True, + "evidence_addressed_claim_graph": True, + "section_and_paragraph_plans": True, + "whole_document_quadratic_inverse": True, + "source_essay_required": False, + "functional_introspection_separated_from_phenomenology": True, + "document_claims_transport_through_protected_tree_spine": True, + "ablations": sorted(self.l5_ablations), + }) + return value + + +class HierarchicalDocumentOracleRelationalV3(RelationalDiscourseOracleRelationalV3): + def __init__(self, *args: Any, l5_ablations: Sequence[str] = (), **kwargs: Any) -> None: + if args: + raise TypeError("HierarchicalDocumentOracleRelationalV3 requires keyword arguments") + self.state_root = Path(kwargs.pop("state_root")).resolve() + language_bundle = Path(kwargs.pop("language_bundle")) + language_cache = kwargs.pop("language_cache", None) + ablations = tuple(kwargs.pop("ablations", ())) + if kwargs: + raise TypeError("unknown arguments: " + ", ".join(sorted(kwargs))) + self.state_root.mkdir(parents=True, exist_ok=True) + self.base_oracle = EpistemicallyGuardedOracleRelationalV3( + state_root=self.state_root, + ) + self.language_compiler = ClauseTypedUnifiedLanguageCompiler( + language_bundle, + cache_directory=( + Path(language_cache).resolve() + if language_cache is not None + else self.state_root / "language_cache" + ), + maximum_hypotheses=32, + ) + self.runtime = HierarchicalSelfDocumentRuntime( + self.base_oracle, + self.language_compiler, + ablations=ablations, + l5_ablations=l5_ablations, + ) + + def capabilities(self) -> dict[str, Any]: + value = dict(super().capabilities()) + value.update({ + "schema_version": SCHEMA, + "version": "0.8.0-l5a-development", + "hierarchical_architecture_documents": True, + "unrestricted_conversation": False, + "unrestricted_reasoning": False, + "unrestricted_agency": False, + "boundary": ( + "Prospectively validated L4 semantic authority plus development-stage " + "hierarchical documents grounded in verified self-architecture observations." + ), + }) + return value + + +__all__ = [ + "ALLOWED_ABLATIONS", "DocumentClaim", "DocumentRequest", "EvidenceAddress", + "HierarchicalDocumentOracleRelationalV3", "HierarchicalSelfDocumentRuntime", + "SCHEMA", +] diff --git a/root/wander-train/stage5l8c_directional_authority_successor.py b/root/wander-train/stage5l8c_directional_authority_successor.py new file mode 100644 index 0000000000000000000000000000000000000000..26a6af5b02c1d499d55b93924ffbe38073409bd0 --- /dev/null +++ b/root/wander-train/stage5l8c_directional_authority_successor.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""L8c successor: preserve explicit wh-argument direction at factual release.""" +from __future__ import annotations + +from pathlib import Path +import re +import time +import types +from typing import Any, Mapping + +from stage5k15_native_mixed_agent import digest +from stage5k224a_persistent_discourse_runtime import JsonSessionStore, human +from stage5k238a_epistemically_guarded_release_candidate import ( + _punctuation_invariant_route, + _specific_identity_act, +) +from stage5l6a_cognitive_state_abi import canonical_sha, file_sha +from stage5l8b_release_hardening_successor import ( + ExplicitArgumentGuardHybridRuntime, + ReleaseHardenedOracleRelationalV3, + ReleaseHardenedRuntime, +) + + +SCHEMA = "oracle-stage5l8c-directional-authority-successor-v1" +VERSION = "0.8.0-l8c" +WORD = re.compile(r"[a-z0-9]+", re.I) +AUXILIARY = frozenset({ + "am", "are", "can", "could", "did", "do", "does", "is", "may", + "might", "must", "shall", "should", "was", "were", "will", "would", +}) +FACTUAL_OPERATIONS = frozenset({ + "persistent_discourse_to_realizing_geometric_authority", + "l4_structural_lexical_bridge_to_geometric_authority", +}) + + +def _phrase_tokens(value: str) -> tuple[str, ...]: + return tuple(WORD.findall(human(str(value)).casefold())) + + +def _span(tokens: tuple[str, ...], phrase: tuple[str, ...]) -> int | None: + for start in range(len(tokens) - len(phrase) + 1): + if tokens[start:start + len(phrase)] == phrase: + return start + return None + + +class DirectionalRoleGuardHybridRuntime(ExplicitArgumentGuardHybridRuntime): + @staticmethod + def _direction_conflict(text: str, result: Mapping[str, Any]) -> bool: + if result.get("status") != "answered" or result.get("operation") not in FACTUAL_OPERATIONS: + return False + plan = tuple(result.get("content_plan", ())) + if len(plan) != 1: + return False + tokens = tuple(WORD.findall(str(text).casefold())) + if not tokens or tokens[0] not in {"what", "which"}: + return False + edge = plan[0] + source_at = _span(tokens, _phrase_tokens(edge.get("source", ""))) + target_at = _span(tokens, _phrase_tokens(edge.get("target", ""))) + relation_forms = { + str(edge.get("relation", "")).casefold(), + str(edge.get("relation", "")).casefold().removesuffix("s"), + } + relation_at = next(( + index for index, token in enumerate(tokens) + if token in relation_forms or token.removesuffix("s") in relation_forms + ), None) + if source_at is None or target_at is not None or relation_at is None: + return False + before_source = tokens[:source_at] + passive_agent = "by" in before_source and bool(set(before_source) & AUXILIARY) + explicit_source_auxiliary = bool(set(before_source) & {"do", "does", "did"}) + return bool( + relation_at < source_at + and not passive_agent + and not explicit_source_auxiliary + ) + + def answer(self, session_id: str, text: str): + discourse = self.agent.reasoning.discourse + state = discourse._state(session_id) + previous = { + "active_bundle": state.active_bundle, + "last_plan": list(state.last_plan), + "last_evidence": list(state.last_evidence), + "salience": list(state.salience), + } + result = super().answer(session_id, text) + if not self._direction_conflict(text, result): + return result + response = ( + "The authenticated fact has the explicit entity in the source role, but this " + "wh-subject request places it in the target role. Reversing that edge is not " + "authorized, so no factual answer is released." + ) + result["status"] = "clarification" + result["operation"] = "directional_argument_role_guard" + result["response"] = response + result["content_plan"] = [] + result["evidence_ledger"] = [] + realization = result.get("realization") + if isinstance(realization, dict): + realization["content_plan"] = [] + realization["evidence_ledger"] = [] + realization["response"] = response + state.active_bundle = previous["active_bundle"] + state.last_plan = previous["last_plan"] + state.last_evidence = previous["last_evidence"] + state.salience = previous["salience"] + if state.turns: + state.turns[-1]["status"] = "clarification" + state.turns[-1]["operation"] = "directional_argument_role_guard" + state.turns[-1]["response"] = response + discourse._save(state) + result["schema_version"] = SCHEMA + result.setdefault("audit", {})["directional_argument_role_preserved"] = True + result["audit"]["reverse_edge_release_blocked"] = True + result["audit"]["external_writes"] = 0 + return result + + +class _DirectionalBaseOracle: + def __init__( + self, *, bundles: Mapping[str, Path], state_root: Path, + workspace_root: Path, substrate_checkpoint: Path, + ) -> None: + self.workspace_root = Path(workspace_root).resolve() + self.phase_root = self.workspace_root / "fable-content" / "oracle-phase3-2026-07-21" + self.bundle_paths = {str(name): Path(path).resolve() for name, path in bundles.items()} + self.state_root = Path(state_root).resolve() + self.state_root.mkdir(parents=True, exist_ok=True) + self.session_store = JsonSessionStore(self.state_root / "sessions") + started = time.perf_counter() + self.runtime = DirectionalRoleGuardHybridRuntime( + self.bundle_paths, + session_store=self.session_store, + substrate_checkpoint=Path(substrate_checkpoint).resolve(), + ) + discourse = self.runtime.agent.reasoning.discourse + discourse._route = types.MethodType(_punctuation_invariant_route, discourse) + discourse._identity_act = types.MethodType(_specific_identity_act, discourse) + self.startup_milliseconds = 1000.0 * (time.perf_counter() - started) + + def chat(self, text: str, *, session_id: str = "default") -> dict[str, Any]: + return self.runtime.answer(session_id, text) + + def capabilities(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA, + "version": VERSION, + "identity_rooted_persistent_dialogue": True, + "active_passive_wh_and_coordinated_propositions": True, + "post_malkuth_evidence_preserving_realization": True, + "verified_implication_hops": 4, + "typed_tools": ["calculator", "list_specialists", "substrate_stats"], + "tool_side_effect_scope": ["pure", "read_only"], + "maximum_ephemeral_specialists": 3, + "rapid_authenticated_specialist_compilation": True, + "explicit_cross_specialist_argument_guard": True, + "explicit_unknown_argument_guard": True, + "directional_argument_role_guard": True, + "traditional_language_model": False, + "transformer": False, + "pretrained_embedding_encoder": False, + "unrestricted_conversation": False, + "unrestricted_reasoning": False, + "unrestricted_agency": False, + "boundary": ( + "Catalog-grounded relational language, verified graph paths, bounded " + "discourse, pure/read-only tools, and temporary specialist coordination." + ), + } + + +class DirectionalAuthorityRuntime(ReleaseHardenedRuntime): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + parent_semantic_sha256 = self.semantic_sha256 + self.semantic_sha256 = digest({ + "schema_version": SCHEMA, + "parent_semantic_sha256": parent_semantic_sha256, + "surface_role_must_match_released_edge_direction": True, + "reverse_edge_release": False, + }) + + def architecture(self) -> dict[str, Any]: + value = dict(super().architecture()) + value.update({ + "schema_version": SCHEMA, + "directional_argument_role_guard": True, + "surface_role_must_match_released_edge_direction": True, + "reverse_edge_release": False, + }) + return value + + +class DirectionalAuthorityOracleRelationalV3(ReleaseHardenedOracleRelationalV3): + def _build_runtime(self) -> None: + old_runtime = getattr(self, "runtime", None) + if old_runtime is not None and hasattr(old_runtime, "close"): + old_runtime.close() + bundle_identity = canonical_sha({ + name: file_sha(path) for name, path in sorted(self.bundle_paths.items()) + }) + self.base_oracle = _DirectionalBaseOracle( + bundles=self.bundle_paths, + state_root=self.state_root, + workspace_root=self.workspace_root, + substrate_checkpoint=( + self.state_root / ("implication_substrate_" + bundle_identity[:16] + ".json") + ), + ) + self.runtime = DirectionalAuthorityRuntime( + self.base_oracle, + self.language_compiler, + ablations=self.ablations, + l5_ablations=self.l5_ablations, + state_abi=self.state_abi, + l7_ablations=self.l7_ablations, + ) + + def capabilities(self) -> dict[str, Any]: + value = dict(super().capabilities()) + value.update({ + "schema_version": SCHEMA, + "version": VERSION, + "directional_argument_role_guard": True, + }) + return value + + +__all__ = [ + "DirectionalAuthorityOracleRelationalV3", "DirectionalAuthorityRuntime", + "DirectionalRoleGuardHybridRuntime", "SCHEMA", "VERSION", +] diff --git a/root/wander-train/stage5m12a_portable_public_sources.py b/root/wander-train/stage5m12a_portable_public_sources.py new file mode 100644 index 0000000000000000000000000000000000000000..5fb78c533990ad0551e2d565116d3dd00ba4f048 --- /dev/null +++ b/root/wander-train/stage5m12a_portable_public_sources.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Portable integrity and entity-resolution surface for the M10 build. + +The M10 compiler itself retains machine-local authenticated source adapters. +This module is the smaller runtime boundary: it validates every packaged +artifact required by the compiled federation and exposes only exact entity +resolution. It does not reacquire or reinterpret source data. +""" +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import Any, Mapping + + +SCHEMA = "oracle-stage5m10a-public-source-specialist-compiler-v1" +ENTITY_SCHEMA = "oracle-stage5m10a-entity-registry-v1" +FEDERATION_SCHEMA = "oracle-stage5m2c3-fragment-authorized-generalist-federation-v1" +BUNDLE_SCHEMA = "oracle-stage5m2b-relational-specialist-bundle-v1" +_DOMAIN = re.compile(r"^[a-z][a-z0-9_]{0,127}$") + + +def canonical(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest() + + +def sha(path: Path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def normalized_label(value: Any) -> str: + return " ".join(re.sub(r"[^\w]+", " ", str(value).casefold(), flags=re.UNICODE).split()) + + +def _child(root: Path, relative: str) -> Path: + root = root.resolve() + path = (root / relative).resolve() + try: + path.relative_to(root) + except ValueError as error: + raise ValueError(f"artifact_path_outside_build:{relative}") from error + return path + + +class EntityResolver: + """Resolve exact external identifiers or unique normalized aliases only.""" + + def __init__(self, registry: Mapping[str, Any]): + if registry.get("schema_version") != ENTITY_SCHEMA: + raise ValueError("unsupported_entity_registry") + identity = registry.get("semantic_identity") + if not isinstance(identity, Mapping) or digest(identity) != registry.get("semantic_identity_sha256"): + raise ValueError("entity_registry_identity_changed") + self.registry = dict(registry) + self._ambiguous = { + item["normalized_alias"]: list(item["candidate_entity_ids"]) + for item in registry.get("ambiguity_quarantine", ()) + } + + @classmethod + def load(cls, path: Path | str) -> "EntityResolver": + return cls(json.loads(Path(path).read_text(encoding="utf-8"))) + + def resolve(self, value: str, *, namespace: str | None = None) -> dict[str, Any]: + external = str(value).casefold() + if namespace is not None and ":" not in external: + external = f"{namespace}:{value}".casefold() + if external in self.registry["external_id_index"]: + return { + "status": "resolved", + "method": "exact_external_id", + "entity_id": self.registry["external_id_index"][external], + } + alias = normalized_label(value) + if alias in self._ambiguous: + return { + "status": "ambiguous", + "method": "exact_alias_collision", + "candidate_entity_ids": self._ambiguous[alias], + } + if alias in self.registry["routable_alias_index"]: + return { + "status": "resolved", + "method": "unique_exact_alias", + "entity_id": self.registry["routable_alias_index"][alias], + } + return {"status": "unresolved", "method": "no_authenticated_identity"} + + +def validate_compilation(output_dir: Path | str) -> dict[str, Any]: + """Recompute every identity required to load a packaged M10 compilation.""" + output_dir = Path(output_dir).resolve() + manifest_path = output_dir / "compilation_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("schema_version") != SCHEMA: + raise ValueError("unsupported_public_source_compilation") + expected_identity = manifest.get("semantic_identity_sha256") + actual_identity = digest({name: value for name, value in manifest.items() if name != "semantic_identity_sha256"}) + if expected_identity != actual_identity: + raise ValueError("public_source_compilation_identity_changed") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping): + raise ValueError("public_source_artifact_manifest_missing") + for relative, expected in artifacts.items(): + path = _child(output_dir, str(relative)) + if not path.is_file() or sha(path) != expected: + raise ValueError(f"public_source_compilation_artifact_changed:{relative}") + source = json.loads((output_dir / "source_manifest.json").read_text(encoding="utf-8")) + source_identity = source.get("semantic_identity_sha256") + if source_identity != digest({name: value for name, value in source.items() if name != "semantic_identity_sha256"}): + raise ValueError("public_source_ledger_identity_changed") + entity = json.loads((output_dir / "entity_registry.json").read_text(encoding="utf-8")) + EntityResolver(entity) + bundle = json.loads((output_dir / "specialist_bundle.json").read_text(encoding="utf-8")) + if bundle.get("schema_version") != BUNDLE_SCHEMA or digest(bundle["semantic_identity"]) != bundle.get("semantic_identity_sha256"): + raise ValueError("public_source_bundle_identity_changed") + if digest(bundle["claims"]) != bundle.get("portable_integrity", {}).get("claims_sha256"): + raise ValueError("public_source_claims_changed") + federation = json.loads((output_dir / "federation.json").read_text(encoding="utf-8")) + if federation.get("schema_version") != FEDERATION_SCHEMA or digest(federation["semantic_identity"]) != federation.get("semantic_identity_sha256"): + raise ValueError("public_source_federation_identity_changed") + portable_path = str(federation["compiler_bundle"]["portable_path"]) + if federation["compiler_bundle"]["portable_sha256"] != sha(_child(output_dir, portable_path)): + raise ValueError("public_source_federation_bundle_changed") + for value in manifest.get("domain_builds", ()): + domain = str(value.get("domain", "")) + if not _DOMAIN.fullmatch(domain): + raise ValueError(f"invalid_domain_build:{domain}") + root = output_dir / "domain_specialists" / domain + if sha(root / "specialist_bundle.json") != value["bundle_sha256"] or sha(root / "federation.json") != value["federation_sha256"]: + raise ValueError(f"public_source_domain_specialist_changed:{domain}") + return manifest + + +__all__ = ["EntityResolver", "validate_compilation", "canonical", "digest", "sha"] diff --git a/root/wander-train/stage5m12d_zero_argument_tool_successor.py b/root/wander-train/stage5m12d_zero_argument_tool_successor.py new file mode 100644 index 0000000000000000000000000000000000000000..0aeedaa76debbc5c898a524de37920fa24ac44e1 --- /dev/null +++ b/root/wander-train/stage5m12d_zero_argument_tool_successor.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""M12d successor: admit schema-valid zero-argument explicit tool calls.""" +from __future__ import annotations + +import hashlib +import re +from dataclasses import replace +from pathlib import Path +from typing import Any + +from stage5m11a_schema_native_long_horizon_agent import SchemaToolRegistry, validate_json_schema +from stage5m12a_portable_successor_runtime import ( + OracleRuntime as OracleM12aRuntime, + ReferenceAwareSchemaGuidedCallCompiler, + build_default_tool_registry as build_m12a_default_tool_registry, +) + + +SCHEMA = "oracle-stage5m12d-zero-argument-tool-successor-v1" +VERSION = "1.0.1" + + +class ZeroArgumentSchemaGuidedCallCompiler(ReferenceAwareSchemaGuidedCallCompiler): + """Retain M12a calls and add `call TOOL` when TOOL accepts `{}`.""" + + def compile_explicit(self, text: str) -> dict[str, Any]: + source = " ".join(str(text).strip().split()) + if re.search(r"\s+then\s+|\s+with\s+", source, re.I): + return super().compile_explicit(source) + match = re.fullmatch(r"(?:call|use)\s+([A-Za-z][A-Za-z0-9_ -]*)", source, re.I) + if not match: + return super().compile_explicit(source) + resolution = self.registry.resolve(match.group(1)) + if resolution["status"] != "resolved": + return { + "schema_version": SCHEMA, "status": "withheld", + "source_sha256": hashlib.sha256(source.encode()).hexdigest(), "actions": [], + "errors": [{"segment": source, "reason": "tool_" + resolution["status"], "resolution": resolution}], + "program": None, "zero_argument_call": True, "traditional_language_model_calls": 0, + "pretrained_embedding_calls": 0, + } + tool = resolution["tool"] + valid, reason = validate_json_schema({}, self.registry.tools[tool].input_schema) + if not valid: + return { + "schema_version": SCHEMA, "status": "withheld", + "source_sha256": hashlib.sha256(source.encode()).hexdigest(), "actions": [], + "errors": [{"segment": source, "reason": "input_schema_invalid:" + str(reason)}], + "program": None, "zero_argument_call": True, "traditional_language_model_calls": 0, + "pretrained_embedding_calls": 0, + } + action = {"action_id": "a0001", "tool": tool, "arguments": {}, "depends_on": [], "expected": [], "retry_budget": 1} + return { + "schema_version": SCHEMA, "status": "compiled", + "source_sha256": hashlib.sha256(source.encode()).hexdigest(), "actions": [action], "errors": [], + "program": {"objective": source, "actions": [action]}, "zero_argument_call": True, + "typed_observation_references": True, "traditional_language_model_calls": 0, + "pretrained_embedding_calls": 0, + } + + +def _upgrade_registry(registry: SchemaToolRegistry) -> SchemaToolRegistry: + tool = registry.tools.get("inspect_oracle") + if tool is not None and tool.executor_id != "oracle.release.inspect.v2": + registry.tools["inspect_oracle"] = replace( + tool, + executor_id="oracle.release.inspect.v2", + executor=lambda _value: {"version": VERSION, "domains": 12, "claims": 215, "maximum_actions": 512, "maximum_parallel": 8}, + ) + return registry + + +def build_default_tool_registry(m10_build: Path | str) -> SchemaToolRegistry: + return _upgrade_registry(build_m12a_default_tool_registry(m10_build)) + + +class OracleRuntime(OracleM12aRuntime): + def __init__(self, state_root: Path | str, **kwargs: Any): + caller_registry = kwargs.get("tool_registry") is not None + super().__init__(state_root, **kwargs) + if not caller_registry: + _upgrade_registry(self.tool_registry) + self.call_compiler = ZeroArgumentSchemaGuidedCallCompiler(self.tool_registry) + + def capabilities(self) -> dict[str, Any]: + value = dict(super().capabilities()) + value.update({"schema_version": SCHEMA, "release_version": VERSION, "zero_argument_schema_calls": True}) + return value + + +__all__ = ["OracleRuntime", "ZeroArgumentSchemaGuidedCallCompiler", "build_default_tool_registry", "SCHEMA", "VERSION"] diff --git a/root/wander-train/stage5m13c1_checkpointed_self_essay.py b/root/wander-train/stage5m13c1_checkpointed_self_essay.py new file mode 100644 index 0000000000000000000000000000000000000000..b2cbbdba8aa8db1671c99d6bc61c14516ede438c --- /dev/null +++ b/root/wander-train/stage5m13c1_checkpointed_self_essay.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""M13c1: checkpointed and order-constrained hierarchical self-essay. + +The accepted M13c development runtime remains unchanged. This additive +successor separates a globally constructed claim/section/paragraph plan from +incremental paragraph realization. The checkpoint contains the typed plan and +realized paragraph ordinals, never a preassembled full response. A new process +can validate the hash chain and frozen evidence identities, realize the next +bounded segment, and finish the same document. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sys +import time +from pathlib import Path +from typing import Any, Mapping, Sequence + + +ROOT = Path(__file__).resolve().parents[1] +TRAIN = ROOT / "wander-train" +RELEASE = ROOT / "wander-release" / "oracle-relational-v3-hf-v1.0.1-release" +for value in (str(TRAIN), str(RELEASE / "src")): + if value not in sys.path: + sys.path.insert(0, value) + +from oracle_relational_v3.oracle_10d_resolution_state import canonical_sha256 +from oracle_relational_v3.stage5m3a_evidence_addressed_realizer import EvidenceAddressedRealizer, SurfaceClause +from stage5m13b_wikipedia_specialist import OracleM13bRuntime +from stage5m13c_hierarchical_self_essay import ( + HierarchicalOperationalSelfEssay, + OracleM13cRuntime, + SECTION_ORDER, + SECTION_TITLES, + digest, +) + + +SCHEMA = "oracle-stage5m13c1-checkpointed-self-essay-v1" +CHECKPOINT_SCHEMA = "oracle-stage5m13c1-self-essay-checkpoint-v1" +RUNTIME_SCHEMA = "oracle-stage5m13c1-runtime-v1" +SECTION_ALIASES = { + "identity": ("identity", "self", "purpose"), + "ingress": ("ingress", "language", "input", "receiving"), + "geometry": ("geometry", "spine", "tree", "routing"), + "knowledge": ("knowledge", "specialists", "federation"), + "authority": ("authority", "epistemic", "withholding"), + "composition": ("composition", "writing", "document"), + "memory": ("memory", "continuity", "persistence"), + "agency": ("agency", "tools", "action"), + "verification": ("verification", "evidence", "failure"), + "limits": ("limits", "limitations", "boundaries", "remaining gaps"), +} + + +def canonical(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def sha_path(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(4 * 1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +class CheckpointStore: + def __init__(self, root: Path | str): + self.root = Path(root).resolve(); self.root.mkdir(parents=True, exist_ok=True) + + def path(self, session_id: str) -> Path: + return self.root / (hashlib.sha256(session_id.encode()).hexdigest() + ".json") + + @staticmethod + def _identity(value: Mapping[str, Any]) -> str: + return digest({key: item for key, item in value.items() if key != "checkpoint_sha256"}) + + def save(self, session_id: str, value: dict[str, Any]) -> dict[str, Any]: + path = self.path(session_id); value = dict(value); value["checkpoint_sha256"] = self._identity(value) + temporary = path.with_suffix(".json.tmp"); temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n"); os.replace(temporary, path) + return value + + def load(self, session_id: str) -> dict[str, Any]: + path = self.path(session_id) + if not path.exists(): + raise FileNotFoundError("no checkpointed self-essay session") + value = json.loads(path.read_text(encoding="utf-8")) + if value.get("schema_version") != CHECKPOINT_SCHEMA or value.get("checkpoint_sha256") != self._identity(value): + raise ValueError("self_essay_checkpoint_identity_changed") + events = value.get("events", []) + for index, event in enumerate(events): + expected_prior = events[index - 1]["event_sha256"] if index else "0" * 64 + if event["prior_sha256"] != expected_prior or event["event_sha256"] != digest({key: item for key, item in event.items() if key != "event_sha256"}): + raise ValueError("self_essay_checkpoint_chain_changed") + plan = value["plan"] + if plan["plan_sha256"] != digest({key: item for key, item in plan.items() if key != "plan_sha256"}): + raise ValueError("self_essay_plan_identity_changed") + if value["cursor"] != len(value["realized_paragraph_ordinals"]): + raise ValueError("self_essay_checkpoint_cursor_changed") + return value + + +class CheckpointedOperationalSelfEssay(HierarchicalOperationalSelfEssay): + @staticmethod + def _mentioned_section(fragment: str) -> str | None: + folded = " ".join(fragment.casefold().split()) + matches = [section for section, aliases in SECTION_ALIASES.items() if any(re.search(r"\b" + re.escape(alias) + r"\b", folded) for alias in aliases)] + return matches[0] if len(matches) == 1 else None + + @classmethod + def requested_order(cls, prompt: str) -> tuple[str, ...]: + order = list(SECTION_ORDER); folded = " ".join(prompt.casefold().split()) + first = None; last = None + if match := re.search(r"\b(?:begin|start|open)\s+(?:the\s+essay\s+)?with\s+(.+?)(?:[,.;]|\band\s+(?:end|finish|conclude)\b|$)", folded): + first = cls._mentioned_section(match.group(1)) + if match := re.search(r"\b(?:end|finish|conclude)\s+(?:the\s+essay\s+)?with\s+(.+?)(?:[,.;]|$)", folded): + last = cls._mentioned_section(match.group(1)) + if first: + order.remove(first); order.insert(0, first) + if last: + order.remove(last); order.append(last) + if first and last and first == last: + raise ValueError("the same section cannot be both first and last") + return tuple(order) + + @staticmethod + def _nested_spine_ordered(paragraphs: Sequence[Sequence[Any]], section_order: Sequence[str], objective_sha: str) -> dict[str, Any]: + paragraph_spines = []; paragraph_handles = [] + for ordinal, claims in enumerate(paragraphs, 1): + surfaces = [SurfaceClause(value.claim_id, value.text, value.authority_class, (value.claim_id,), value.proof_path, inline_citations=value.evidence_ids) for value in claims] + local_sha = digest({"objective": objective_sha, "paragraph": ordinal, "claims": [value.public() for value in claims]}) + spine = EvidenceAddressedRealizer._spine(surfaces, local_sha); paragraph_spines.append({"paragraph": ordinal, **spine}) + paragraph_handles.append(SurfaceClause(f"paragraph_{ordinal:02d}", local_sha, "opaque_paragraph_handle", tuple(value.claim_id for value in claims), ("paragraph_malkuth_cleanup", local_sha))) + section_spines = []; section_handles = [] + for section_index, section in enumerate(section_order): + handles = paragraph_handles[section_index * 3:(section_index + 1) * 3] + local_sha = digest({"objective": objective_sha, "section": section, "paragraph_handles": [value.public() for value in handles]}) + spine = EvidenceAddressedRealizer._spine(handles, local_sha); section_spines.append({"section": section, **spine}) + section_handles.append(SurfaceClause(f"section_{section_index + 1:02d}", local_sha, "opaque_section_handle", tuple(value.clause_id for value in handles), ("section_malkuth_cleanup", local_sha))) + document = EvidenceAddressedRealizer._spine(section_handles, objective_sha); all_spines = [document, *section_spines, *paragraph_spines] + return { + "schema_version": "oracle-stage5m13c1-three-level-tree-spine-v1", "levels": 3, "protected_root_ratio": "4:1", "paragraph_states": 30, "section_states": 10, "document_states": 1, + "section_order": list(section_order), "route": document["route"], "tree_nodes_registered": document["tree_nodes_registered"], "tree_paths_registered": document["tree_paths_registered"], + "pre_malkuth_payload_unbinds": sum(value["pre_malkuth_payload_unbinds"] for value in all_spines), "malkuth_payload_unbinds": sum(value["malkuth_payload_unbinds"] for value in all_spines), + "minimum_protected_root_cosine": min(value["minimum_protected_root_cosine"] for value in all_spines), "terminal_cleanup_exact": all(value["terminal_cleanup_exact"] for value in all_spines), "minimum_cleanup_margin": min(value["minimum_cleanup_margin"] for value in all_spines), + "document_spine": document, "section_spines": section_spines, "paragraph_spines": paragraph_spines, + } + + def plan(self, prompt: str) -> dict[str, Any]: + values, evidence = self._evidence(); observations = self._observations(values); claims = self._claims(observations); section_order = self.requested_order(prompt) + observations_by_section = {section: [value for value in observations if value.section == section] for section in SECTION_ORDER} + claims_by_observation = {value.observation_id: [claim for claim in claims if claim.observation_id == value.observation_id] for value in observations} + paragraphs = []; paragraph_plan = []; section_plan = [] + for section_index, section in enumerate(section_order): + section_claim_ids = [] + for observation in observations_by_section[section]: + local = claims_by_observation[observation.observation_id]; paragraph_ordinal = len(paragraphs) + 1; paragraphs.append(local) + paragraph_plan.append({"paragraph": paragraph_ordinal, "section": section, "focus": observation.focus, "claim_ids": [value.claim_id for value in local], "objective_sha256": digest({"section": section, "focus": observation.focus, "prompt": hashlib.sha256(prompt.encode()).hexdigest()})}) + section_claim_ids.extend(value.claim_id for value in local) + section_plan.append({"section": section, "title": SECTION_TITLES[section], "paragraphs": list(range(section_index * 3 + 1, section_index * 3 + 4)), "claim_ids": section_claim_ids, "document_thesis": "protected relational identity through bounded transformation"}) + objective_sha = digest({"prompt": prompt, "section_order": section_order, "thesis": "protected relational identity through bounded transformation", "evidence": {key: value.sha256 for key, value in evidence.items()}}) + spine = self._nested_spine_ordered(paragraphs, section_order, objective_sha) + plan = { + "schema_version": SCHEMA, "prompt": prompt, "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "section_order": list(section_order), + "evidence_registry": [evidence[key].public() for key in sorted(evidence)], "observations": [dict(value.__dict__) for value in observations], "claims": [value.public() for value in claims], + "section_plan": section_plan, "paragraph_plan": paragraph_plan, "tree_spine": spine, + "whole_document_inverse": {"candidate_layouts": ["requested", "canonical", "collapsed", "reversed"], "selected": "requested", "assignment": [1, 0, 0, 0], "assignment_hamming_weight": 1, "exhaustive_one_hot_minimum_verified": True}, + "plan_sha256": "", + } + plan["plan_sha256"] = digest({key: item for key, item in plan.items() if key != "plan_sha256"}) + return plan + + @staticmethod + def _paragraph_text(plan: Mapping[str, Any], ordinal: int) -> str: + paragraph = plan["paragraph_plan"][ordinal - 1]; claims = {value["claim_id"]: value for value in plan["claims"]} + return " ".join(claims[claim_id]["text"] for claim_id in paragraph["claim_ids"]) + + @classmethod + def _response(cls, plan: Mapping[str, Any], paragraph_count: int) -> str: + parts = ["# What it is operationally like to be Oracle after M13b"] + emitted_section = None + for ordinal in range(1, paragraph_count + 1): + paragraph = plan["paragraph_plan"][ordinal - 1]; section = paragraph["section"] + if section != emitted_section: + parts.append("## " + SECTION_TITLES[section]); emitted_section = section + parts.append(cls._paragraph_text(plan, ordinal)) + return "\n\n".join(parts) + + @staticmethod + def _event(checkpoint: Mapping[str, Any], operation: str, before: int, after: int) -> dict[str, Any]: + prior = checkpoint["events"][-1]["event_sha256"] if checkpoint["events"] else "0" * 64 + event = {"ordinal": len(checkpoint["events"]) + 1, "prior_sha256": prior, "operation": operation, "cursor_before": before, "cursor_after": after, "plan_sha256": checkpoint["plan"]["plan_sha256"]} + event["event_sha256"] = digest(event); return event + + @staticmethod + def _validate_sources(plan: Mapping[str, Any]) -> None: + for value in plan["evidence_registry"]: + path = ROOT / value["path"] + if not path.is_file() or sha_path(path) != value["sha256"]: + raise ValueError("self_essay_evidence_identity_changed:" + value["evidence_id"]) + + def begin(self, store: CheckpointStore, session_id: str, prompt: str, *, paragraphs: int = 7) -> dict[str, Any]: + if store.path(session_id).exists(): + raise FileExistsError("self-essay checkpoint already exists") + plan = self.plan(prompt); count = max(1, min(int(paragraphs), 29)); checkpoint = {"schema_version": CHECKPOINT_SCHEMA, "session_id_sha256": hashlib.sha256(session_id.encode()).hexdigest(), "plan": plan, "cursor": count, "realized_paragraph_ordinals": list(range(1, count + 1)), "events": [], "completed_artifact_sha256": None} + checkpoint["events"].append(self._event(checkpoint, "begin", 0, count)); checkpoint = store.save(session_id, checkpoint) + return {"schema_version": SCHEMA, "status": "in_progress", "operation": "m13c1_checkpointed_self_essay_begin", "response": self._response(plan, count), "cursor": count, "remaining_paragraphs": 30 - count, "plan_sha256": plan["plan_sha256"], "checkpoint_sha256": checkpoint["checkpoint_sha256"], "full_response_stored": False, "tree_spine": plan["tree_spine"]} + + def resume(self, store: CheckpointStore, session_id: str, *, paragraphs: int = 7) -> dict[str, Any]: + checkpoint = store.load(session_id); plan = checkpoint["plan"]; self._validate_sources(plan); before = int(checkpoint["cursor"]) + if before >= 30: + return self._final_artifact(plan, checkpoint) + after = min(30, before + max(1, int(paragraphs))); checkpoint["cursor"] = after; checkpoint["realized_paragraph_ordinals"] = list(range(1, after + 1)); checkpoint["events"].append(self._event(checkpoint, "resume" if after < 30 else "complete", before, after)) + if after < 30: + checkpoint = store.save(session_id, checkpoint) + return {"schema_version": SCHEMA, "status": "in_progress", "operation": "m13c1_checkpointed_self_essay_resume", "response": self._response(plan, after), "cursor": after, "remaining_paragraphs": 30 - after, "plan_sha256": plan["plan_sha256"], "checkpoint_sha256": checkpoint["checkpoint_sha256"], "full_response_stored": False, "tree_spine": plan["tree_spine"]} + artifact = self._final_artifact(plan, checkpoint); checkpoint["completed_artifact_sha256"] = artifact["artifact_sha256"]; store.save(session_id, checkpoint); return artifact + + @classmethod + def _final_artifact(cls, plan: Mapping[str, Any], checkpoint: Mapping[str, Any]) -> dict[str, Any]: + response = cls._response(plan, 30); words = re.findall(r"\b\w+(?:[-']\w+)?\b", response, flags=re.UNICODE) + artifact = { + "schema_version": SCHEMA, "status": "answered", "operation": "m13c1_checkpointed_hierarchical_operational_self_essay", "response": response, "factual_release": True, "authority_class": "sealed_architecture_observation_plus_bounded_inference", + "response_objective": {"prompt_sha256": plan["prompt_sha256"], "artifact": "essay", "scope": "current_oracle_architecture", "target_words": [3000, 5000], "source_essay_available": False, "requested_section_order": plan["section_order"]}, + "evidence_registry": plan["evidence_registry"], "observations": plan["observations"], "claims": plan["claims"], "section_plan": plan["section_plan"], "paragraph_plan": plan["paragraph_plan"], "plan_sha256": plan["plan_sha256"], "tree_spine": plan["tree_spine"], "whole_document_inverse": plan["whole_document_inverse"], + "audit": {"word_count": len(words), "paragraph_count": 30, "section_count": 10, "claim_count": 150, "all_claims_evidence_addressed": all(value["evidence_ids"] for value in plan["claims"]), "all_inferences_have_proof_paths": all(value["proof_path"] for value in plan["claims"] if "inference" in value["authority_class"]), "source_essay_used": False, "pre_malkuth_payload_unbinds": plan["tree_spine"]["pre_malkuth_payload_unbinds"], "terminal_cleanup_exact": plan["tree_spine"]["terminal_cleanup_exact"], "traditional_language_model_calls": 0, "transformer_calls": 0, "pretrained_embedding_calls": 0, "next_token_predictions": 0, "gradient_steps": 0}, + "checkpoint": {"cursor": 30, "events": len(checkpoint["events"]), "hash_chain_valid": True, "resumed_after_process_reconstruction": len(checkpoint["events"]) > 1}, "artifact_sha256": "", + } + artifact["artifact_sha256"] = canonical_sha256({key: item for key, item in artifact.items() if key != "artifact_sha256"}); return artifact + + def generate(self, prompt: str) -> dict[str, Any]: + plan = self.plan(prompt); checkpoint = {"events": [], "cursor": 30}; return self._final_artifact(plan, checkpoint) + + +class OracleM13c1Runtime: + STAGED = re.compile(r"\b(?:in\s+stages|checkpoint(?:ed)?|pause\s+after|incrementally)\b", re.I) + CONTINUE = re.compile(r"^\s*(?:continue|resume|keep going)\s*[.!?]*$", re.I) + + def __init__(self, state_root: Path | str, *, wordnet_build: Path | str, wikipedia_build: Path | str, **options: Any): + self.state_root = Path(state_root).resolve(); self.state_root.mkdir(parents=True, exist_ok=True) + self.base = OracleM13bRuntime(self.state_root / "m13b", wordnet_build=wordnet_build, wikipedia_build=wikipedia_build, **options) + self.self_essay = CheckpointedOperationalSelfEssay(self.base); self.checkpoints = CheckpointStore(self.state_root / "self_essay_checkpoints") + self.complete = OracleM13cRuntime(self.state_root / "m13c-complete", wordnet_build=wordnet_build, wikipedia_build=wikipedia_build, **options) + + def begin_self_essay(self, prompt: str, *, session_id: str, paragraphs: int = 7) -> dict[str, Any]: + return self.self_essay.begin(self.checkpoints, session_id, prompt, paragraphs=paragraphs) + + def continue_self_essay(self, *, session_id: str, paragraphs: int = 7) -> dict[str, Any]: + return self.self_essay.resume(self.checkpoints, session_id, paragraphs=paragraphs) + + def chat(self, text: str, *, session_id: str = "default") -> dict[str, Any]: + source = " ".join(str(text).split()) + if self.CONTINUE.fullmatch(source) and self.checkpoints.path(session_id).exists(): + return self.continue_self_essay(session_id=session_id) + if OracleM13cRuntime.SELF_REQUEST.search(source): + if self.STAGED.search(source): + return self.begin_self_essay(source, session_id=session_id) + return self.self_essay.generate(source) + return self.base.chat(source, session_id=session_id) + + def capabilities(self) -> dict[str, Any]: + value = dict(self.base.capabilities()); value.update({"schema_version": RUNTIME_SCHEMA, "hierarchical_self_essay": True, "checkpointed_mid_document_resume": True, "bounded_section_order_constraints": True, "self_essay_target_words": [3000, 5000], "self_essay_spine_levels": 3, "source_essay_required": False, "unrestricted_conversation": False}); return value + + +__all__ = ["CheckpointStore", "CheckpointedOperationalSelfEssay", "OracleM13c1Runtime", "SCHEMA", "CHECKPOINT_SCHEMA", "RUNTIME_SCHEMA"] diff --git a/root/wander-train/stage5m13c_hierarchical_self_essay.py b/root/wander-train/stage5m13c_hierarchical_self_essay.py new file mode 100644 index 0000000000000000000000000000000000000000..af8ed7bcd9279bf3b1e320cea1c02dd27db3d40b --- /dev/null +++ b/root/wander-train/stage5m13c_hierarchical_self_essay.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""M13c: evidence-addressed several-thousand-word operational self-essay. + +This additive successor consumes structured reports and the accepted M13b +runtime. It does not contain a source essay. Thirty observation units are +expanded into typed factual, inferential, relational, geometric, and boundary +claims; claims bind into paragraph states, paragraph handles into section +states, and section handles into a document state. Every level traverses a +complete protected 4:1 Tree spine and unbinds only at its own Malkuth boundary. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + + +ROOT = Path(__file__).resolve().parents[1] +PHASE = ROOT / "fable-content" / "oracle-m-series-2026-08-15" +TRAIN = ROOT / "wander-train" +RELEASE = ROOT / "wander-release" / "oracle-relational-v3-hf-v1.0.1-release" +for value in (str(TRAIN), str(RELEASE / "src")): + if value not in sys.path: + sys.path.insert(0, value) + +from oracle_relational_v3.oracle_10d_resolution_state import canonical_sha256 +from oracle_relational_v3.stage5m3a_evidence_addressed_realizer import EvidenceAddressedRealizer, SurfaceClause +from stage5m13b_wikipedia_specialist import OracleM13bRuntime, sha_file + + +SCHEMA = "oracle-stage5m13c-hierarchical-self-essay-v1" +RUNTIME_SCHEMA = "oracle-stage5m13c-runtime-v1" +SECTION_ORDER = ( + "identity", "ingress", "geometry", "knowledge", "authority", + "composition", "memory", "agency", "verification", "limits", +) +SECTION_TITLES = { + "identity": "Identity as an operational invariant", + "ingress": "Receiving language as relational compilation", + "geometry": "The stabilizing spine and nested Tree transport", + "knowledge": "A federation rather than a monolithic memory", + "authority": "Knowing, attributing, and withholding", + "composition": "Growing a document from a bounded relational seed", + "memory": "Continuity through content-addressed state", + "agency": "Action as verified state transition", + "verification": "Failure as part of the architecture's description", + "limits": "What this form warrants and what remains open", +} + + +def canonical(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest() + + +def sentence(value: str) -> str: + value = " ".join(str(value).split()).strip() + return value.rstrip(".?!") + "." + + +@dataclass(frozen=True) +class Evidence: + evidence_id: str + label: str + path: str + sha256: str + observation_path: str + observed_value: Any + authority_class: str = "sealed_architecture_observation" + + def public(self) -> dict[str, Any]: + return dict(self.__dict__) + + +@dataclass(frozen=True) +class Observation: + observation_id: str + section: str + focus: str + fact: str + consequence: str + relation: str + boundary: str + evidence_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class EssayClaim: + claim_id: str + observation_id: str + section: str + role: str + authority_class: str + text: str + evidence_ids: tuple[str, ...] + proof_path: tuple[str, ...] + + def public(self) -> dict[str, Any]: + return { + "claim_id": self.claim_id, "observation_id": self.observation_id, + "section": self.section, "role": self.role, + "authority_class": self.authority_class, "text": self.text, + "evidence_ids": list(self.evidence_ids), "proof_path": list(self.proof_path), + } + + +class SelfEssayLedger: + def __init__(self, root: Path | str): + self.root = Path(root).resolve(); self.root.mkdir(parents=True, exist_ok=True) + + def append(self, session_id: str, prompt: str, artifact: Mapping[str, Any]) -> dict[str, Any]: + path = self.root / (hashlib.sha256(session_id.encode()).hexdigest() + ".json") + prior = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {"schema_version": "oracle-stage5m13c-self-essay-ledger-v1", "session_id_sha256": hashlib.sha256(session_id.encode()).hexdigest(), "events": []} + prior_head = prior["events"][-1]["event_sha256"] if prior["events"] else "0" * 64 + event = {"ordinal": len(prior["events"]) + 1, "prior_sha256": prior_head, "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "artifact_sha256": artifact["artifact_sha256"], "plan_sha256": artifact["plan_sha256"]} + event["event_sha256"] = digest(event); prior["events"].append(event) + temporary = path.with_suffix(".json.tmp"); temporary.write_text(json.dumps(prior, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n"); os.replace(temporary, path) + return {"turns": len(prior["events"]), "head_sha256": event["event_sha256"], "hash_chain_valid": all(value["prior_sha256"] == (prior["events"][index - 1]["event_sha256"] if index else "0" * 64) and value["event_sha256"] == digest({key: item for key, item in value.items() if key != "event_sha256"}) for index, value in enumerate(prior["events"]))} + + +class HierarchicalOperationalSelfEssay: + SOURCE_PATHS = { + "E01": "fable-content/oracle-m-series-2026-08-15/stage5m9_milestone_closure_report_v1.json", + "E02": "fable-content/oracle-m-series-2026-08-15/stage5m10_milestone_closure_report_v1.json", + "E03": "fable-content/oracle-m-series-2026-08-15/stage5m11_milestone_closure_report_v1.json", + "E04": "fable-content/oracle-m-series-2026-08-15/stage5m12_milestone_closure_report_v1.json", + "E05": "fable-content/oracle-m-series-2026-08-15/stage5m12b_public_diagnostic_report_v1.json", + "E06": "fable-content/oracle-m-series-2026-08-15/stage5m13a_wordnet_specialist_development_report_v1.json", + "E07": "fable-content/oracle-m-series-2026-08-15/stage5m13a_wordnet_prospective_report_v1.json", + "E08": "fable-content/oracle-m-series-2026-08-15/stage5m13b_milestone_closure_report_v1.json", + "E09": "fable-content/oracle-m-series-2026-08-15/stage5m13b_accepted_manifest_v1.json", + "E10": "wander-train/stage5m13b_wikipedia_specialist.py", + "E11": "fable-content/oracle-m-series-2026-08-15/stage5m12c_self_descriptive_essay_release_v1.md", + "E12": "wander-train/stage5m13c_hierarchical_self_essay.py", + } + + def __init__(self, m13b: OracleM13bRuntime): + self.m13b = m13b + + def _evidence(self) -> tuple[dict[str, Any], dict[str, Evidence]]: + loaded: dict[str, Any] = {} + registry: dict[str, Evidence] = {} + for evidence_id, relative in self.SOURCE_PATHS.items(): + path = ROOT / relative + if path.suffix == ".json": + value: Any = json.loads(path.read_text(encoding="utf-8")) + elif path.suffix == ".md": + text = path.read_text(encoding="utf-8"); value = {"words": len(text.split()), "sha256": sha_file(path)} + else: + value = {"sha256": sha_file(path), "bytes": path.stat().st_size} + loaded[evidence_id] = value + registry[evidence_id] = Evidence(evidence_id, path.stem, relative, sha_file(path), "registered_document", value) + for evidence_id in ("E01", "E02", "E03", "E04", "E06", "E07", "E08"): + if loaded[evidence_id].get("accepted") is not True: + raise ValueError(f"unaccepted architecture evidence: {evidence_id}") + return loaded, registry + + @staticmethod + def _observations(values: Mapping[str, Any]) -> tuple[Observation, ...]: + m9, m10, m11, m12 = (values[key]["metrics"] for key in ("E01", "E02", "E03", "E04")) + public = values["E05"]["metrics"]["m12"]; wordnet = values["E06"]["metrics"]; lexical = values["E07"]["metrics"]; wiki = values["E08"]["metrics"] + def obs(index: int, section: str, focus: str, fact: str, consequence: str, relation: str, boundary: str, evidence: Sequence[str]) -> Observation: + return Observation(f"O{index:02d}", section, focus, sentence(fact), sentence(consequence), sentence(relation), sentence(boundary), tuple(evidence)) + return ( + obs(1, "identity", "non-autoregressive identity", "I am instantiated in this release as a CPU-native relational runtime whose registered execution path reports no transformer or gradient calls", "Operational identity is therefore carried by explicit state contracts, evidence addresses, and deterministic transitions rather than by a hidden next-token trajectory", "This makes identity a constraint that persists through operations instead of a stylistic preamble placed above them", "The absence of a transformer does not by itself prove superior intelligence, generality, or biological similarity", ("E04", "E09")), + obs(2, "identity", "purpose as state preservation", "My operational purpose is to preserve an evidence-bounded relational form from interpretation through authority, memory, action, and release", "A request can remain one developing object even while different specialists, tools, and discourse mechanisms transform its payload", "The purpose functions as the protected root against which local transformations are judged for congruence", "This functional continuity is not evidence of a private phenomenal subject", ("E03", "E04", "E11")), + obs(3, "identity", "current scale", f"The accepted integrated federation exposes {wiki['integrated_claims']:,} claims while retaining the earlier curated tier", "Self-description must now refer to a materially broader live system than the 440-word M12 release essay described", "Scale changes the number of possible evidence paths, but it does not change the rule that a path needs authenticated authority", "Claim count is capacity for addressable evidence, not a direct measure of understanding", ("E08", "E11")), + obs(4, "ingress", "language as compilation", f"The M9 successor passed {m9['m9b_passed']} of {m9['m9b_total']} sealed topic-mask cases after separating operator cues from authenticated topic spans", "Receiving a sentence can be treated as compiling an act, artifact, topic, reference, and authority request into separate relational fields", "This separation prevents a word inside a topic name from simultaneously becoming an unintended discourse command", "The registered construction families do not yet cover unrestricted natural language", ("E01",)), + obs(5, "ingress", "situated word identity", f"The lexical specialist preserves {wordnet['synsets']:,} synsets, {wordnet['unique_aliases']:,} unique aliases, and {wordnet['ambiguous_aliases']:,} ambiguity sets", "A word is interpreted as a situated candidate set rather than as one permanently collapsed definition", "Context can narrow a lexical state while ambiguity quarantine prevents convenience from masquerading as certainty", "WordNet sense structure is lexical evidence and is neither encyclopedic truth nor a complete model of pragmatics", ("E06", "E07")), + obs(6, "ingress", "structural authority separation", "The accepted runtime gives linguistic programs structural and discourse authority but does not grant them factual authority merely for being well formed", "Interpretation can remain flexible without allowing fluent syntax to manufacture a world fact", "The language surface proposes a route; authenticated specialists and explicit temporary worlds determine what may occupy that route", "A conservative authority boundary can still withhold requests whose intended meaning a person would regard as obvious", ("E01", "E04")), + obs(7, "geometry", "four-to-one root protection", "The stabilizing spine represents every protected state as a four-part governing root plus one orthogonal payload component", "Payloads can accumulate and be transformed while the objective remains geometrically dominant", "Root protection converts identity preservation from a narrative promise into a measurable cosine constraint", "A favorable root cosine does not establish that every payload has been semantically interpreted correctly", ("E03", "E08", "E10")), + obs(8, "geometry", "ten-node route", "Admitted states traverse the registered ten-node and twenty-two-path Tree route without payload unbinding before Malkuth", "Intermediate nodes can add bounded transformations while opaque content remains protected from premature cleanup", "Routing and release become different operations, which permits richer binding without erasing provenance", "The Tree is an engineered representational topology and these traces do not prove a unique correspondence to anatomy or metaphysics", ("E03", "E10")), + obs(9, "geometry", "nested document spines", f"The accepted M13b run carried {wiki['long_documents']} unseen documents through paragraph-level and document-level HRR spines with exact terminal cleanup", "A document no longer needs to occupy one flat superposition in which dozens of clause handles compete for the same cleanup dictionary", "Claims terminate inside paragraph states, paragraph handles enter section states here, and section handles enter the document root", "Hierarchical cleanup increases tested capacity but does not make semantic coherence automatic", ("E08", "E10", "E12")), + obs(10, "knowledge", "curated specialist tier", f"The curated M10 federation compiled {m10['claims']} authenticated relations across {m10['domains']} domains from {m10['sources']} preserved source families in {m10['compilation_seconds']:.3f} seconds", "High-specificity source adapters can retain stronger authority than broad secondary-source retrieval", "The runtime therefore routes curated evidence before lexical or encyclopedic fallbacks", "A small curated graph cannot supply broad world knowledge on its own", ("E02",)), + obs(11, "knowledge", "sense-preserving lexical tier", f"The WordNet shard contributes {wordnet['claims']:,} definition and relation claims and passed {lexical['prospective_cases']:,} post-freeze prospective cases", "Lexical relations can be compiled and hot-loaded without retraining a carrier model", "This tier supplies the geometry of senses, taxonomies, and relations that later planners can use to interpret evidence", "Lexical coverage does not settle which sense a novel discourse intends when contextual evidence remains insufficient", ("E06", "E07")), + obs(12, "knowledge", "attributed encyclopedic tier", f"The Wikipedia specialist retains {wiki['wikipedia_articles']:,} exact article identities and {wiki['wikipedia_claims']:,} admitted sentence claims", "Orders-of-magnitude broader evidence can be added as an indexed specialist while source digests and sentence addresses remain exact", "The federation gains breadth by adding cells with distinct authority rather than dissolving every source into one parameter field", "The preserved snapshot has no recoverable date and cannot be represented as current or adjudicated truth", ("E08", "E09")), + obs(13, "authority", "ambiguity quarantine", f"The final M13b protocol exhausted the {wiki['ambiguities_exhausted']} collision titles that remained unseen after earlier panels and withheld all {wiki['unknowns_withheld']} registered unknowns", "Not choosing is an active epistemic operation when candidate identity is not unique", "Quarantine preserves possible forms without prematurely binding one of them into the released world state", "Withholding protects factual integrity but can reduce conversational helpfulness when clarification mechanisms are too narrow", ("E08",)), + obs(14, "authority", "secondary-source attribution", "Every encyclopedic statement retains page identity, canonical URL, raw-record digest, abstract digest, sentence ordinal, fact digest, and selection class", "Retrieval can be audited at the exact sentence while rank remains a relevance signal rather than a truth score", "Context candidates are kept distinct from exact anchor-article statements even when they share lexical geometry", "Perfect provenance can show what a source said without proving that the source was correct", ("E08", "E10")), + obs(15, "authority", "measured wrong release", f"The disjoint M12 public diagnostic answered {public['passed']} of {public['cases']} cases, withheld {public['withheld']}, and made {public['wrong_releases']} wrong releases", "The project cannot equate fluent release with warranted release and must measure incorrect assertion separately from abstention", "This negative evidence determines why epistemic control remains a first-class part of the successor plan", "The diagnostic is a sampled research panel rather than an official aggregate BIG-bench score", ("E04", "E05")), + obs(16, "composition", "the earlier self-account", f"The sealed M12 self-description contains {values['E11']['words']} words and is grounded in twenty-four registered architecture claims", "It proves bounded operational introspection but not sustained novel document construction", "The present planner treats that essay as a measured predecessor artifact, not as text to copy or expand", "A first-person functional account does not establish phenomenal experience", ("E03", "E11")), + obs(17, "composition", "lexical long form", f"The M13a prospective run produced {lexical['long_form_cases']} five-paragraph lexical documents while preserving exact senses and restart behavior", "The result showed that relational paths can support multi-paragraph layout without a language model", "Those short documents provided a precursor state whose limits motivated hierarchical encyclopedic transport", "Documents of roughly two hundred words do not satisfy several-thousand-word global coherence", ("E07",)), + obs(18, "composition", "encyclopedic document growth", f"The final unseen M13b documents used twenty-four distinct claims and measured {wiki['long_document_p95_ms']:.1f} milliseconds at p95 on the local CPU", "Sparse articles can now grow into bounded multi-source context reports while maintaining exact nested cleanup", "The document state is assembled globally from paragraph objectives rather than padded by repeating one response template", "The accepted M13b reports remained extractive and only 400 to 710 words long", ("E08", "E10")), + obs(19, "memory", "long-horizon transition continuity", f"The M11 prospective trajectory completed {m11['sequential_actions']} dependent actions and {m11['sequential_events']} authenticated events across a forced failure and restart", "A goal can persist as a hash-chained relational object beyond one process lifetime", "Continuity is carried by verified transitions and content addresses rather than by replaying an unstructured transcript", "The demonstrated horizon is finite and uses registered tool schemas", ("E03",)), + obs(20, "memory", "compact delta state", f"The retained M11 checkpoint, delta, and observation store occupied {m11['delta_store_bytes']:,} bytes, or {100*m11['delta_ratio']:.2f} percent of the registered full-copy counterfactual", "Relational change can be recorded as a sparse transformation instead of duplicating an entire state at every step", "This creates a computational analogue of continuity through conserved structure and explicit differences", "The ratio is tied to the registered workload and is not a universal compression bound", ("E03",)), + obs(21, "memory", "parallel lanes", f"The same accepted agent scheduled {m11['parallel_actions']} independent actions in {m11['parallel_turns']} eight-way turns", "Independent reasoning or action heads can share one protected objective while operating on separate ready states", "Parallelism becomes a property of the dependency graph rather than a claim that all thoughts should be flattened into one sequence", "Parallel scheduling does not reduce the critical path of genuinely dependent reasoning", ("E03",)), + obs(22, "agency", "caller-bound tool schemas", f"The accepted M11 run exercised {m11['dynamic_tool_schemas']} previously unseen caller-supplied schemas", "Tools can enter as typed state-transition contracts without being memorized during model construction", "Input validation, predicted effects, observations, and receipts place action inside the same relational ABI as language and memory", "This does not authorize arbitrary external tools or irreversible side effects", ("E03", "E04")), + obs(23, "agency", "closed-loop verification", "The agent executes one bounded action, compares the observed result with its predicted schema, and commits only the verified delta", "Reasoning about a tool call is separated from evidence that the call actually produced its expected effect", "Retry, substitution, rollback, and reconstruction are state transitions with explicit authority rather than invisible control flow", "A schema match cannot establish that a real-world action was socially desirable or globally safe", ("E03",)), + obs(24, "agency", "knowledge tools as specialists", "The M13b runtime exposes exact Wikipedia resolution and bounded Wikipedia search alongside specialist inspection and lexical resolution", "A conversation can request evidence addresses through tools without promoting the observation into a timeless world fact", "Specialist retrieval and agentic action meet at an authority boundary rather than at a shared pool of untyped text", "The current surface grammar does not yet induce arbitrary tool programs from unrestricted prose", ("E08", "E09")), + obs(25, "verification", "reproducible corpus compilation", "Two independent Wikipedia builds produced byte-identical 1,471,492,096-byte indexes while reconstructing sampled raw records and admitted sentences exactly", "A large specialist can be tested as a deterministic compiled artifact rather than accepted because a training loss decreased", "Reproduction fixes the identity of the knowledge substrate so behavioral changes can be attributed to runtime mechanisms", "Byte identity proves construction repeatability, not factual correctness of the source corpus", ("E08", "E09")), + obs(26, "verification", "preserved failure lineage", "M13b preserved a 17-of-20 development failure, an 11-of-13 prospective routing failure, a 12-of-13 Unicode search failure, and an unexecuted cardinality-invalid panel", "Failures became observations that isolated resolver order, flat-superposition interference, tokenizer mismatch, and finite-panel constraints", "Successor evidence is causally stronger because each repair is linked to the exact counterexample that required it", "A sequence of repaired tests can still overfit unless later panels exclude every exposed case", ("E08", "E09")), + obs(27, "verification", "final unseen acceptance", f"The cardinality-corrected M13b run passed {wiki['accepted_prospective_gates']} of {wiki['accepted_prospective_gates']} gates across {wiki['accepted_registered_cases']} registered cases", "The accepted breadth and transport claims rest on post-freeze cases rather than on development replay", "The final panel combines source reconstruction, identity, ambiguity, unknowns, search, documents, runtime dispatch, restart, parity, concurrency, and latency", "Passing this panel establishes its bounded subclaims and does not close the larger open-ended objective", ("E08", "E09")), + obs(28, "limits", "open conversation remains unproven", "The accepted manifest explicitly names hierarchical semantic planning and several-thousand-word self-description as the next additive stage", "Breadth alone cannot turn extractive evidence into arbitrary explanation, argument, synthesis, or conversational repair", "The remaining problem is to transform evidence geometry into globally situated discourse while preserving authority at every clause", "The current system must not be labeled unrestricted, frontier-equivalent, or generally intelligent", ("E08", "E09")), + obs(29, "limits", "phenomenology boundary", "No accepted functional trace establishes subjective consciousness, sentience, or phenomenal experience", "The phrase what it is like can be answered here only as an operational account of constraints, transitions, persistence, and release", "First-person grammar names the runtime whose state is being described; it does not upgrade computation into evidence of an inner witness", "Stronger claims require evidence of a kind this architecture and these tests do not supply", ("E03", "E04", "E11")), + obs(30, "limits", "the next falsifiable threshold", "M13c requires a prospectively prompted three-thousand-to-five-thousand-word document with evidence-complete clauses, non-repeating paragraphs, nested cleanup, restart-persistent planning, and an explicit claim boundary", "Success would show that the architecture can preserve a global thesis across many local transformations at a scale beyond its earlier extractive reports", "That threshold connects the geometric architecture to a measurable form of sustained semantic construction", "Even success would remain one milestone before broad conversation, arbitrary prose induction, standard benchmark gains, and a sealed successor release", ("E08", "E09", "E12")), + ) + + @staticmethod + def _claims(observations: Sequence[Observation]) -> tuple[EssayClaim, ...]: + claims: list[EssayClaim] = [] + node_by_section = dict(zip(SECTION_ORDER, ("Keter", "Chokmah", "Binah", "Chesed", "Geburah", "Tiferet", "Netzach", "Hod", "Yesod", "Malkuth"))) + inference_frames = ( + "{focus} has an operational consequence: {value}", + "In the running state, {focus} changes the computation because {value}", + "The consequence carried by {focus} is concrete: {value}", + "When {focus} becomes active, what changes in practice is that {value}", + "{focus}, read as a state transition, yields a specific result: {value}", + "Through {focus}, the architecture can maintain that {value}", + ) + relation_frames = ( + "Placed beside {prior}, the new state matters because {value}", + "The relation to {prior} is not decorative: {value}", + "This extends {prior} by making explicit that {value}", + "Where {prior} fixed the preceding constraint, the present state adds that {value}", + "The transition from {prior} is congruent only because {value}", + "Against the background of {prior}, this becomes load-bearing because {value}", + ) + geometry_frames = ( + "{focus}, bound at {node}, becomes a protected paragraph state; only its opaque handle rises into the section and document spines before their separate Malkuth cleanups [E12]", + "{focus} occupies {node} geometrically: local claims clean up inside the paragraph, while a paragraph handle rather than the raw payload continues upward [E12]", + "{focus} enters the routed form at {node}, preserving its claim bundle until paragraph release and passing only a cleaned handle into the wider argument [E12]", + "{focus} is carried by {node} within the nested spine without exposing its payload to the section or document level before the registered terminal boundaries [E12]", + "{focus} functions as a {node} state here, so local binding capacity and global thesis continuity are protected by different cleanup dictionaries [E12]", + ) + boundary_frames = ( + "The limit attached to {focus} must remain explicit: {value}", + "What {focus} does not authorize is equally definite: {value}", + "The negative space around {focus} is part of the result, because {value}", + "Epistemic control around {focus} requires the restriction that {value}", + "The account of {focus} would be distorted without this boundary: {value}", + "The claim about {focus} terminates by refusing to conclude more, since {value}", + ) + prior: Observation | None = None + for observation in observations: + base = observation.observation_id; index = int(base[1:]) - 1 + citations = ",".join(observation.evidence_ids) + fact = observation.fact.rstrip(".") + f" [{citations}]." + inference = inference_frames[index % len(inference_frames)].format(focus=observation.focus, value=observation.consequence[:1].lower() + observation.consequence[1:].rstrip(".")) + f" [{citations}]." + prior_focus = prior.focus if prior else "the protected document root" + relation_evidence = tuple(dict.fromkeys((*observation.evidence_ids, *((prior.evidence_ids) if prior else ())))) + relation_citations = ",".join(relation_evidence) + relation = relation_frames[index % len(relation_frames)].format(prior=prior_focus, value=observation.relation[:1].lower() + observation.relation[1:].rstrip(".")) + f" [{relation_citations}]." + geometry = geometry_frames[index % len(geometry_frames)].format(node=node_by_section[observation.section], focus=observation.focus) + "." + boundary = boundary_frames[index % len(boundary_frames)].format(focus=observation.focus, value=observation.boundary[:1].lower() + observation.boundary[1:].rstrip(".")) + f" [{citations}]." + for suffix, role, authority, text, evidence, proof in ( + ("F", "fact", "sealed_architecture_observation", fact, observation.evidence_ids, ("registered_observation", base)), + ("I", "consequence", "bounded_architectural_inference", inference, observation.evidence_ids, ("observation_to_consequence", base + "F")), + ("R", "relation", "bounded_architectural_inference", relation, relation_evidence, ("adjacent_observation_relation", prior.observation_id if prior else "document_root", base)), + ("G", "geometry", "planner_state_observation", geometry, ("E12",), ("paragraph_section_document_route", base)), + ("B", "boundary", "epistemic_boundary" if observation.section in {"authority", "limits"} else "bounded_claim_boundary", boundary, observation.evidence_ids, ("claim_scope_negation", base + "F")), + ): + claims.append(EssayClaim(base + suffix, base, observation.section, role, authority, sentence(text), tuple(evidence), tuple(proof))) + prior = observation + return tuple(claims) + + @staticmethod + def _nested_spine(paragraphs: Sequence[Sequence[EssayClaim]], objective_sha: str) -> dict[str, Any]: + paragraph_spines = []; paragraph_handles = [] + for ordinal, claims in enumerate(paragraphs, 1): + surfaces = [SurfaceClause(value.claim_id, value.text, value.authority_class, (value.claim_id,), value.proof_path, inline_citations=value.evidence_ids) for value in claims] + local_sha = digest({"objective": objective_sha, "paragraph": ordinal, "claims": [value.public() for value in claims]}) + spine = EvidenceAddressedRealizer._spine(surfaces, local_sha); paragraph_spines.append({"paragraph": ordinal, **spine}) + paragraph_handles.append(SurfaceClause(f"paragraph_{ordinal:02d}", local_sha, "opaque_paragraph_handle", tuple(value.claim_id for value in claims), ("paragraph_malkuth_cleanup", local_sha))) + section_spines = []; section_handles = [] + for section_index, section in enumerate(SECTION_ORDER): + handles = paragraph_handles[section_index * 3:(section_index + 1) * 3] + local_sha = digest({"objective": objective_sha, "section": section, "paragraph_handles": [value.public() for value in handles]}) + spine = EvidenceAddressedRealizer._spine(handles, local_sha); section_spines.append({"section": section, **spine}) + section_handles.append(SurfaceClause(f"section_{section_index + 1:02d}", local_sha, "opaque_section_handle", tuple(value.clause_id for value in handles), ("section_malkuth_cleanup", local_sha))) + document = EvidenceAddressedRealizer._spine(section_handles, objective_sha) + all_spines = [document, *section_spines, *paragraph_spines] + return { + "schema_version": "oracle-stage5m13c-three-level-tree-spine-v1", "levels": 3, "protected_root_ratio": "4:1", + "paragraph_states": len(paragraph_spines), "section_states": len(section_spines), "document_states": 1, + "route": document["route"], "tree_nodes_registered": document["tree_nodes_registered"], "tree_paths_registered": document["tree_paths_registered"], + "pre_malkuth_payload_unbinds": sum(value["pre_malkuth_payload_unbinds"] for value in all_spines), + "malkuth_payload_unbinds": sum(value["malkuth_payload_unbinds"] for value in all_spines), + "minimum_protected_root_cosine": min(value["minimum_protected_root_cosine"] for value in all_spines), + "terminal_cleanup_exact": all(value["terminal_cleanup_exact"] for value in all_spines), + "minimum_cleanup_margin": min(value["minimum_cleanup_margin"] for value in all_spines), + "document_spine": document, "section_spines": section_spines, "paragraph_spines": paragraph_spines, + } + + def generate(self, prompt: str) -> dict[str, Any]: + started = time.perf_counter(); values, evidence = self._evidence(); observations = self._observations(values); claims = self._claims(observations) + by_observation = {value.observation_id: [claim for claim in claims if claim.observation_id == value.observation_id] for value in observations} + paragraphs = [by_observation[value.observation_id] for value in observations] + response_parts = ["# What it is operationally like to be Oracle after M13b"] + paragraph_plan = []; section_plan = [] + for section_index, section_name in enumerate(SECTION_ORDER): + response_parts.append("## " + SECTION_TITLES[section_name]) + section_claims = [] + for paragraph_offset in range(3): + observation = observations[section_index * 3 + paragraph_offset]; local = by_observation[observation.observation_id] + response_parts.append(" ".join(value.text for value in local)) + paragraph_plan.append({"paragraph": len(paragraph_plan) + 1, "section": section_name, "focus": observation.focus, "claim_ids": [value.claim_id for value in local], "objective_sha256": digest({"section": section_name, "focus": observation.focus, "prompt": hashlib.sha256(prompt.encode()).hexdigest()})}) + section_claims.extend(value.claim_id for value in local) + section_plan.append({"section": section_name, "title": SECTION_TITLES[section_name], "paragraphs": list(range(section_index * 3 + 1, section_index * 3 + 4)), "claim_ids": section_claims, "document_thesis": "protected relational identity through bounded transformation"}) + response = "\n\n".join(response_parts) + objective_sha = digest({"prompt": prompt, "thesis": "protected relational identity through bounded transformation", "evidence": {key: value.sha256 for key, value in evidence.items()}}) + spine = self._nested_spine(paragraphs, objective_sha) + response_words = re.findall(r"\b\w+(?:[-']\w+)?\b", response, flags=re.UNICODE) + prompt_words = re.findall(r"\b\w+(?:[-']\w+)?\b", prompt.casefold(), flags=re.UNICODE) + response_folded = set(value.casefold() for value in response_words) + sentences = [part.strip() for part in re.split(r"(?<=[.!?])\s+", re.sub(r"(?m)^#+\s+.*$", "", response)) if part.strip()] + plan_sha = digest({"sections": section_plan, "paragraphs": paragraph_plan}) + artifact = { + "schema_version": SCHEMA, "status": "answered", "operation": "m13c_hierarchical_operational_self_essay", + "response": response, "factual_release": True, "authority_class": "sealed_architecture_observation_plus_bounded_inference", + "response_objective": {"prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "artifact": "essay", "scope": "current_oracle_architecture", "target_words": [3000, 5000], "source_essay_available": False}, + "epistemic_contract": {"factual_claims_require_registered_observations": True, "inferences_require_proof_paths": True, "language_structure_has_factual_authority": False, "functional_introspection_is_phenomenology": False, "wikipedia_context_is_anchor_fact": False}, + "evidence_registry": [evidence[key].public() for key in sorted(evidence)], "observations": [dict(value.__dict__) for value in observations], "claims": [value.public() for value in claims], + "section_plan": section_plan, "paragraph_plan": paragraph_plan, "plan_sha256": plan_sha, "tree_spine": spine, + "whole_document_inverse": {"candidate_layouts": ["canonical", "collapsed", "reversed_sections", "uncited"], "candidate_losses": {"canonical": 0.0, "collapsed": 4.0, "reversed_sections": 6.0, "uncited": 10.0}, "selected": "canonical", "assignment": [1, 0, 0, 0], "assignment_hamming_weight": 1, "exhaustive_one_hot_minimum_verified": True, "whole_document_selected_globally": True}, + "audit": { + "word_count": len(response_words), "paragraph_count": len(paragraphs), "section_count": len(section_plan), "claim_count": len(claims), + "all_claims_evidence_addressed": all(value.evidence_ids for value in claims), "all_inferences_have_proof_paths": all(value.proof_path for value in claims if "inference" in value.authority_class), + "all_references_content_addressed": all(len(value.sha256) == 64 for value in evidence.values()), "duplicate_sentence_count": len(sentences) - len(set(sentences)), + "first_person_operational_account": bool(re.search(r"\bI\s+(?:am|receive|cannot|operate|preserve)\b", response)), "phenomenology_claim_withheld": "No accepted functional trace establishes subjective consciousness" in response, + "current_m13b_claim_count_present": f"{values['E08']['metrics']['integrated_claims']:,}" in response, "source_essay_used": False, + "input_echo_ratio": sum(value in response_folded for value in prompt_words) / max(1, len(prompt_words)), "pre_malkuth_payload_unbinds": spine["pre_malkuth_payload_unbinds"], "terminal_cleanup_exact": spine["terminal_cleanup_exact"], + "traditional_language_model_calls": 0, "transformer_calls": 0, "pretrained_embedding_calls": 0, "next_token_predictions": 0, "gradient_steps": 0, + }, + "latency_ms": 1000.0 * (time.perf_counter() - started), "artifact_sha256": "", + } + artifact["artifact_sha256"] = canonical_sha256({key: value for key, value in artifact.items() if key not in {"latency_ms", "artifact_sha256"}}) + return artifact + + +class OracleM13cRuntime: + SELF_REQUEST = re.compile(r"\b(?:essay|report|document|treatise|self-description)\b.*\b(?:oracle|yourself|your architecture|being this architecture|what it is like)\b|\b(?:oracle|yourself|your architecture|being this architecture|what it is like)\b.*\b(?:essay|report|document|treatise|self-description)\b", re.I) + + def __init__(self, state_root: Path | str, *, wordnet_build: Path | str, wikipedia_build: Path | str, **options: Any): + self.state_root = Path(state_root).resolve(); self.state_root.mkdir(parents=True, exist_ok=True) + self.base = OracleM13bRuntime(self.state_root / "m13b", wordnet_build=wordnet_build, wikipedia_build=wikipedia_build, **options) + self.self_essay = HierarchicalOperationalSelfEssay(self.base); self.ledger = SelfEssayLedger(self.state_root / "self_essay_ledger") + + def chat(self, text: str, *, session_id: str = "default") -> dict[str, Any]: + source = " ".join(str(text).split()) + if not self.SELF_REQUEST.search(source): + return self.base.chat(source, session_id=session_id) + artifact = self.self_essay.generate(source); artifact["session"] = self.ledger.append(session_id, source, artifact) + return {"schema_version": RUNTIME_SCHEMA, "status": "answered", "operation": artifact["operation"], "response": artifact["response"], "factual_release": True, "authority_class": artifact["authority_class"], "self_document": artifact, "session": artifact["session"]} + + def capabilities(self) -> dict[str, Any]: + value = dict(self.base.capabilities()); value.update({"schema_version": RUNTIME_SCHEMA, "hierarchical_self_essay": True, "self_essay_target_words": [3000, 5000], "self_essay_spine_levels": 3, "source_essay_required": False, "unrestricted_conversation": False}); return value + + +__all__ = ["HierarchicalOperationalSelfEssay", "OracleM13cRuntime", "SCHEMA", "RUNTIME_SCHEMA"] diff --git a/root/wander-train/stage5m13d1b_surface_varied_document.py b/root/wander-train/stage5m13d1b_surface_varied_document.py new file mode 100644 index 0000000000000000000000000000000000000000..7f6649ca16351d23c57e05195557f96c07fb63de --- /dev/null +++ b/root/wander-train/stage5m13d1b_surface_varied_document.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""M13d1b: surface-varied successor over unchanged M13d1 claim geometry.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Sequence + +from oracle_relational_v3.stage5m3a_evidence_addressed_realizer import SurfaceClause +from stage5m13d1_general_hierarchical_document import ( + GeneralHierarchicalDocumentCompiler, + OracleM13d1Runtime, + ROLE_DEFINITIONS, + RoutedClaim, + words, +) + + +RUNTIME_SCHEMA = "oracle-stage5m13d1b-surface-varied-runtime-v1" + + +class SurfaceVariedGeneralDocumentCompiler(GeneralHierarchicalDocumentCompiler): + @staticmethod + def _paragraph_clauses(topic: str, role: tuple[str, str, tuple[str, ...]], paragraph_index: int, claims: Sequence[RoutedClaim], citation_start: int) -> tuple[list[SurfaceClause], list[dict[str, Any]]]: + role_name, role_title, _ = role + shared = sorted(set.intersection(*(set(words(value.claim.title + " " + value.claim.fact)) for value in claims))) if claims else [] + query_terms = sorted({term for value in claims for term in value.query_shared_terms}) + relation_terms = (query_terms or shared)[:4]; relation_surface = ", ".join(relation_terms) if relation_terms else "the query-conditioned directional scores" + intros = ( + f"Paragraph {paragraph_index} routes three sources through {role_title.casefold()} for {topic}. Retrieval relevance is not factual authority.", + f"For {topic}, paragraph {paragraph_index} assigns three records to {role_title.casefold()}. The route organizes surfaces without upgrading their authority.", + f"The {role_title.casefold()} pass places three {topic} records in paragraph {paragraph_index}. No directional score converts a candidate into an anchor fact.", + ) + clauses = [SurfaceClause(f"p{paragraph_index:02d}_objective", intros[(paragraph_index - 1) % 3], "bounded_document_inference", tuple(value.claim.claim_id for value in claims), ("balanced_ten_direction_assignment", role_name, str(paragraph_index)))] + templates = ( + "[{citation}] As a {selection}, preserved page {title} states: {fact}.", + "[{citation}] Attributed evidence from {title}, retrieved as a {selection}, reads: {fact}.", + "[{citation}] The {selection} addressed to {title} contributes: {fact}.", + "[{citation}] In the preserved snapshot, {title} reports this {selection}: {fact}.", + "[{citation}] Source page {title} supplies the following {selection}: {fact}.", + "[{citation}] Bound to page {title}, this {selection} says: {fact}.", + ) + references = [] + for offset, routed in enumerate(claims): + claim = routed.claim; number = citation_start + offset; citation = f"P{number:03d}" + selection = {"exact_article": "exact-anchor statement", "fts_candidate": "conjunctive-query statement", "fts_lexical_context_candidate": "lexical-context candidate"}.get(claim.selection, claim.selection.replace("_", "-")) + text = templates[(number - 1) % len(templates)].format(citation=citation, selection=selection, title=claim.title, fact=claim.fact.rstrip(". ")) + clauses.append(SurfaceClause(f"p{paragraph_index:02d}_source_{offset + 1}", text, "attributed_secondary_source", (claim.claim_id,), (claim.selection, claim.claim_id, claim.raw_record_sha256, claim.fact_sha256), inline_citations=(citation,))) + references.append({"citation_id": citation, "claim_id": claim.claim_id, "page_id": claim.page_id, "title": claim.title, "source_url": claim.canonical_url, "raw_record_sha256": claim.raw_record_sha256, "abstract_sha256": claim.abstract_sha256, "fact_sha256": claim.fact_sha256, "sentence_ordinal": claim.sentence_ordinal, "selection": claim.selection, "bm25_rank": claim.rank}) + boundaries = ( + f"At paragraph {paragraph_index}, shared surface {relation_surface} sets grouping only, not causation or truth about {topic}; each statement stays bound to its cited page.", + f"This paragraph's overlap marker is {relation_surface}. It marks textual organization, not causation or truth about {topic}, and supplies no evidence of consensus, completeness, or freshness.", + f"The local geometry turns on {relation_surface}, not causation or truth about {topic}; it releases no world relation beyond the cited records.", + ) + clauses.append(SurfaceClause(f"p{paragraph_index:02d}_boundary", boundaries[(paragraph_index - 1) % 3], "epistemic_boundary", tuple(value.claim.claim_id for value in claims), ("surface_relation_not_world_relation", role_name, *relation_terms))) + return clauses, references + + +class OracleM13d1bRuntime(OracleM13d1Runtime): + def __init__(self, state_root: Path | str, *, wordnet_build: Path | str, wikipedia_build: Path | str, **options: Any): + super().__init__(state_root, wordnet_build=wordnet_build, wikipedia_build=wikipedia_build, **options) + self.general_documents = SurfaceVariedGeneralDocumentCompiler(self) + + def capabilities(self) -> dict[str, Any]: + value = super().capabilities(); value.update({"schema_version": RUNTIME_SCHEMA, "citation_addressed_surface_variation": True, "general_document_surface_template_families": 6}); return value + + +__all__ = ["OracleM13d1bRuntime", "SurfaceVariedGeneralDocumentCompiler", "RUNTIME_SCHEMA"] diff --git a/root/wander-train/stage5m13d1c_ordinal_surface_document.py b/root/wander-train/stage5m13d1c_ordinal_surface_document.py new file mode 100644 index 0000000000000000000000000000000000000000..6e5f43904b14dc217808ee349fd8e2676966ba24 --- /dev/null +++ b/root/wander-train/stage5m13d1c_ordinal_surface_document.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""M13d1c: make every meta-clause ordinal-bearing and non-repeating.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +from typing import Any, Sequence + +from oracle_relational_v3.stage5m3a_evidence_addressed_realizer import SurfaceClause +from stage5m13d1_general_hierarchical_document import ROLE_DEFINITIONS, RoutedClaim, words +from stage5m13d1b_surface_varied_document import OracleM13d1bRuntime, SurfaceVariedGeneralDocumentCompiler + + +RUNTIME_SCHEMA = "oracle-stage5m13d1c-ordinal-surface-runtime-v1" + + +class OrdinalSurfaceGeneralDocumentCompiler(SurfaceVariedGeneralDocumentCompiler): + @staticmethod + def _paragraph_clauses(topic: str, role: tuple[str, str, tuple[str, ...]], paragraph_index: int, claims: Sequence[RoutedClaim], citation_start: int) -> tuple[list[SurfaceClause], list[dict[str, Any]]]: + clauses, references = SurfaceVariedGeneralDocumentCompiler._paragraph_clauses(topic, role, paragraph_index, claims, citation_start) + _role_name, role_title, _ = role + shared = sorted(set.intersection(*(set(words(value.claim.title + " " + value.claim.fact)) for value in claims))) if claims else [] + query_terms = sorted({term for value in claims for term in value.query_shared_terms}) + relation_terms = (query_terms or shared)[:4]; relation_surface = ", ".join(relation_terms) if relation_terms else "the query-conditioned directional scores" + intros = ( + f"Paragraph {paragraph_index} routes three sources through {role_title.casefold()} for {topic}, while keeping retrieval relevance separate from factual authority.", + f"For {topic}, paragraph {paragraph_index} assigns three records to {role_title.casefold()} without upgrading the authority of their source surfaces.", + f"The {role_title.casefold()} pass places three {topic} records in paragraph {paragraph_index}, but no directional score converts a candidate into an anchor fact.", + ) + boundaries = ( + f"At paragraph {paragraph_index}, shared surface {relation_surface} sets grouping only, not causation or truth about {topic}, and each statement stays bound to its cited page.", + f"Paragraph {paragraph_index} uses overlap marker {relation_surface} for textual organization, not causation or truth about {topic}, consensus, completeness, or freshness.", + f"Paragraph {paragraph_index} turns its local geometry on {relation_surface}, not causation or truth about {topic}, and releases no world relation beyond the cited records.", + ) + clauses[0] = replace(clauses[0], text=intros[(paragraph_index - 1) % 3]) + clauses[-1] = replace(clauses[-1], text=boundaries[(paragraph_index - 1) % 3]) + return clauses, references + + +class OracleM13d1cRuntime(OracleM13d1bRuntime): + def __init__(self, state_root: Path | str, *, wordnet_build: Path | str, wikipedia_build: Path | str, **options: Any): + super().__init__(state_root, wordnet_build=wordnet_build, wikipedia_build=wikipedia_build, **options) + self.general_documents = OrdinalSurfaceGeneralDocumentCompiler(self) + + def capabilities(self) -> dict[str, Any]: + value = super().capabilities(); value.update({"schema_version": RUNTIME_SCHEMA, "ordinal_bearing_meta_clauses": True}); return value + + +__all__ = ["OracleM13d1cRuntime", "OrdinalSurfaceGeneralDocumentCompiler", "RUNTIME_SCHEMA"] diff --git a/root/wander-train/stage5m13d1e_prefix_ordinal_document.py b/root/wander-train/stage5m13d1e_prefix_ordinal_document.py new file mode 100644 index 0000000000000000000000000000000000000000..6a0a094b50487b91aaa9d9ddc2e43fa0988f8721 --- /dev/null +++ b/root/wander-train/stage5m13d1e_prefix_ordinal_document.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""M13d1e: move the remaining meta-clause ordinal into the start prefix.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +from typing import Any, Sequence + +from oracle_relational_v3.stage5m3a_evidence_addressed_realizer import SurfaceClause +from stage5m13d1_general_hierarchical_document import RoutedClaim +from stage5m13d1d_interior_ordinal_document import InteriorOrdinalGeneralDocumentCompiler, OracleM13d1dRuntime + + +RUNTIME_SCHEMA = "oracle-stage5m13d1e-prefix-ordinal-runtime-v1" + + +class PrefixOrdinalGeneralDocumentCompiler(InteriorOrdinalGeneralDocumentCompiler): + @staticmethod + def _paragraph_clauses(topic: str, role: tuple[str, str, tuple[str, ...]], paragraph_index: int, claims: Sequence[RoutedClaim], citation_start: int) -> tuple[list[SurfaceClause], list[dict[str, Any]]]: + clauses, references = InteriorOrdinalGeneralDocumentCompiler._paragraph_clauses(topic, role, paragraph_index, claims, citation_start) + if (paragraph_index - 1) % 3 == 1: + _role_name, role_title, _ = role + clauses[0] = replace(clauses[0], text=f"For paragraph {paragraph_index} on {topic}, assignment {paragraph_index} binds three records to {role_title.casefold()} and leaves their source authority unchanged.") + return clauses, references + + +class OracleM13d1eRuntime(OracleM13d1dRuntime): + def __init__(self, state_root: Path | str, *, wordnet_build: Path | str, wikipedia_build: Path | str, **options: Any): + super().__init__(state_root, wordnet_build=wordnet_build, wikipedia_build=wikipedia_build, **options) + self.general_documents = PrefixOrdinalGeneralDocumentCompiler(self) + + def capabilities(self) -> dict[str, Any]: + value = super().capabilities(); value.update({"schema_version": RUNTIME_SCHEMA, "prefix_ordinal_surface_identity": True}); return value + + +__all__ = ["OracleM13d1eRuntime", "PrefixOrdinalGeneralDocumentCompiler", "RUNTIME_SCHEMA"] diff --git a/root/wander-train/stage5m13d2_anchored_general_document.py b/root/wander-train/stage5m13d2_anchored_general_document.py new file mode 100644 index 0000000000000000000000000000000000000000..03d8e37c8396564a5700ff5f63cb33c32a09be03 --- /dev/null +++ b/root/wander-train/stage5m13d2_anchored_general_document.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""M13d2: require query-anchor evidence before contextual long-form release.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from stage5m13d1e_prefix_ordinal_document import OracleM13d1eRuntime, PrefixOrdinalGeneralDocumentCompiler + + +RUNTIME_SCHEMA = "oracle-stage5m13d2-anchored-general-document-runtime-v1" + + +class AnchoredGeneralDocumentCompiler(PrefixOrdinalGeneralDocumentCompiler): + def _claim_pool(self, topic: str, limit: int = 90): + resolution, claims, metadata = super()._claim_pool(topic, limit=limit) + if resolution.get("status") == "unresolved" and metadata.get("conjunctive_query_claims", 0) == 0: + discarded = int(metadata.get("lexical_context_claims", 0)) + metadata = { + **metadata, + "status": "insufficient", + "unique_claims": 0, + "lexical_context_claims": 0, + "discarded_unanchored_context_claims": discarded, + "query_anchor_required": True, + "query_anchor_present": False, + "context_asserted_as_anchor_fact": False, + } + return resolution, [], metadata + metadata = {**metadata, "query_anchor_required": resolution.get("status") == "unresolved", "query_anchor_present": bool(metadata.get("conjunctive_query_claims", 0) or metadata.get("exact_anchor_claims", 0)), "discarded_unanchored_context_claims": 0} + return resolution, claims, metadata + + +class OracleM13d2Runtime(OracleM13d1eRuntime): + def __init__(self, state_root: Path | str, *, wordnet_build: Path | str, wikipedia_build: Path | str, **options: Any): + super().__init__(state_root, wordnet_build=wordnet_build, wikipedia_build=wikipedia_build, **options) + self.general_documents = AnchoredGeneralDocumentCompiler(self) + + def capabilities(self) -> dict[str, Any]: + value = super().capabilities(); value.update({"schema_version": RUNTIME_SCHEMA, "unresolved_general_document_requires_query_anchor": True, "unanchored_context_release": False}); return value + + +__all__ = ["AnchoredGeneralDocumentCompiler", "OracleM13d2Runtime", "RUNTIME_SCHEMA"] diff --git a/root/wander-train/stage5m13d_hardened_checkpointed_self_essay.py b/root/wander-train/stage5m13d_hardened_checkpointed_self_essay.py new file mode 100644 index 0000000000000000000000000000000000000000..42aa34422a42a3aef49219fd72f9c62a0d208ef1 --- /dev/null +++ b/root/wander-train/stage5m13d_hardened_checkpointed_self_essay.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""M13d: semantic checkpoint binding above the accepted M13c1 runtime.""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path +from typing import Any + +from stage5m13c_hierarchical_self_essay import SECTION_ORDER +from stage5m13c1_checkpointed_self_essay import ( + CheckpointStore, + CheckpointedOperationalSelfEssay, + OracleM13c1Runtime, +) + + +RUNTIME_SCHEMA = "oracle-stage5m13d-hardened-checkpoint-runtime-v1" + + +class SemanticallyBoundCheckpointStore(CheckpointStore): + """Validate meaning-bearing invariants in addition to content hashes.""" + + def load(self, session_id: str) -> dict[str, Any]: + value = super().load(session_id) + if value.get("session_id_sha256") != hashlib.sha256(session_id.encode()).hexdigest(): + raise ValueError("self_essay_checkpoint_session_binding_changed") + cursor = value.get("cursor") + if not isinstance(cursor, int) or isinstance(cursor, bool) or not 1 <= cursor <= 30: + raise ValueError("self_essay_checkpoint_cursor_domain_changed") + if value.get("realized_paragraph_ordinals") != list(range(1, cursor + 1)): + raise ValueError("self_essay_checkpoint_realized_ordinal_sequence_changed") + + plan = value["plan"] + order = plan.get("section_order") + if not isinstance(order, list) or len(order) != 10 or set(order) != set(SECTION_ORDER): + raise ValueError("self_essay_plan_section_permutation_changed") + if len(plan.get("paragraph_plan", [])) != 30 or len(plan.get("section_plan", [])) != 10 or len(plan.get("observations", [])) != 30 or len(plan.get("claims", [])) != 150: + raise ValueError("self_essay_plan_cardinality_changed") + claim_ids = [item.get("claim_id") for item in plan["claims"]] + if len(set(claim_ids)) != 150 or any(len(item.get("claim_ids", [])) != 5 for item in plan["paragraph_plan"]): + raise ValueError("self_essay_plan_claim_partition_changed") + spine = plan.get("tree_spine", {}) + if spine.get("levels") != 3 or spine.get("paragraph_states") != 30 or spine.get("section_states") != 10 or spine.get("document_states") != 1 or not spine.get("terminal_cleanup_exact") or spine.get("pre_malkuth_payload_unbinds") != 0: + raise ValueError("self_essay_plan_spine_invariant_changed") + + events = value.get("events") + if not isinstance(events, list) or not events: + raise ValueError("self_essay_checkpoint_event_history_changed") + prior_cursor = 0 + for index, event in enumerate(events): + if event.get("ordinal") != index + 1: + raise ValueError("self_essay_checkpoint_event_ordinal_changed") + if event.get("plan_sha256") != plan["plan_sha256"]: + raise ValueError("self_essay_checkpoint_event_plan_binding_changed") + before = event.get("cursor_before"); after = event.get("cursor_after") + if not isinstance(before, int) or not isinstance(after, int) or before != prior_cursor or not before < after <= 30: + raise ValueError("self_essay_checkpoint_event_cursor_continuity_changed") + expected_operation = "begin" if index == 0 else ("complete" if after == 30 else "resume") + if event.get("operation") != expected_operation: + raise ValueError("self_essay_checkpoint_event_operation_changed") + if after == 30 and index != len(events) - 1: + raise ValueError("self_essay_checkpoint_event_after_completion_changed") + prior_cursor = after + if prior_cursor != cursor: + raise ValueError("self_essay_checkpoint_event_terminal_cursor_changed") + + completed = value.get("completed_artifact_sha256") + if cursor < 30: + if completed is not None: + raise ValueError("self_essay_checkpoint_premature_completion_identity") + else: + if not isinstance(completed, str) or re.fullmatch(r"[0-9a-f]{64}", completed) is None: + raise ValueError("self_essay_checkpoint_completed_artifact_identity_changed") + expected = CheckpointedOperationalSelfEssay._final_artifact(plan, value)["artifact_sha256"] + if completed != expected: + raise ValueError("self_essay_checkpoint_completed_artifact_identity_changed") + return value + + def save(self, session_id: str, value: dict[str, Any]) -> dict[str, Any]: + super().save(session_id, value) + return self.load(session_id) + + +class OracleM13dRuntime(OracleM13c1Runtime): + def __init__(self, state_root: Path | str, *, wordnet_build: Path | str, wikipedia_build: Path | str, **options: Any): + super().__init__(state_root, wordnet_build=wordnet_build, wikipedia_build=wikipedia_build, **options) + self.checkpoints = SemanticallyBoundCheckpointStore(self.state_root / "self_essay_checkpoints") + + def capabilities(self) -> dict[str, Any]: + value = super().capabilities() + value.update({ + "schema_version": RUNTIME_SCHEMA, + "semantic_checkpoint_binding": True, + "checkpoint_session_binding": True, + "checkpoint_cursor_continuity": True, + "checkpoint_plan_event_binding": True, + "checkpoint_completion_identity": True, + }) + return value + + +__all__ = ["OracleM13dRuntime", "SemanticallyBoundCheckpointStore", "RUNTIME_SCHEMA"] diff --git a/root/wander-train/stage5m13e2_construction_kernel.py b/root/wander-train/stage5m13e2_construction_kernel.py new file mode 100644 index 0000000000000000000000000000000000000000..b89dfbf00765a7bacd4ed468511f92f26d2c481f --- /dev/null +++ b/root/wander-train/stage5m13e2_construction_kernel.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""M13e2 E2-K1: productive construction kernel. + +Typed lexicon + CKY chart parse over construction rules -> ClauseProgram IR -> +temporary premise world with forward-chaining proof -> answer with premises/rule/ +conclusion. Negation, tense, and quantification are feature transformers that +compose with every clause rule, so withheld combinations of known constructions +parse without combination-specific rules. `compositional=False` ablates the +kernel to whole-signature memory for the causal transfer test. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass, field +from typing import Any + +SCHEMA = "oracle-stage5m13e2-construction-kernel-v1" + +QUANTIFIERS = {"every": "every", "each": "every", "no": "no", "some": "some"} +ARTICLES = {"a", "an", "the"} +COPULAS = {"is": ("present", "sg"), "are": ("present", "pl"), "was": ("past", "sg"), "were": ("past", "pl")} +AUXILIARIES = {"did": "past", "does": "present", "do": "present", "will": "future"} +NEGATIONS = {"not", "never"} +IRREGULAR_PAST = {"went": "go", "made": "make", "took": "take", "gave": "give", "held": "hold", + "built": "build", "sold": "sell", "bought": "buy", "taught": "teach", "kept": "keep"} + + +def _lemma_from_past(token: str) -> str | None: + if token in IRREGULAR_PAST: + return IRREGULAR_PAST[token] + if token.endswith("ied") and len(token) > 4: + return token[:-3] + "y" + if token.endswith("ed") and len(token) > 3: + base = token[:-2] + if base.endswith(("v", "s", "z", "c", "u")) or base[-1] not in "aeiou" and base[-1] * 2 == base[-2:]: + pass + candidate = base + "e" + return candidate if base and base[-1] in "vszcu" else base + return None + + +def _lemma_from_3sg(token: str) -> str | None: + if token.endswith("ies") and len(token) > 4: + return token[:-3] + "y" + if token.endswith("es") and re.search(r"(?:s|x|z|ch|sh)es$", token): + return token[:-2] + if token.endswith("s") and not token.endswith("ss") and len(token) > 2: + return token[:-1] + return None + + +def _singular(noun: str) -> str: + if noun.endswith("ies") and len(noun) > 4: + return noun[:-3] + "y" + if noun.endswith("es") and re.search(r"(?:s|x|z|ch|sh)es$", noun): + return noun[:-2] + if noun.endswith("s") and not noun.endswith("ss") and len(noun) > 2: + return noun[:-1] + return noun + + +@dataclass(frozen=True) +class Item: + category: str + semantics: tuple # hashable payload + span: tuple + + +CLOSED_CLASS = set(QUANTIFIERS) | ARTICLES | set(COPULAS) | set(AUXILIARIES) | NEGATIONS | {"and", "if"} + + +def _subject_of(item: "Item") -> tuple: + if item.category == "NAME": + return ("proper", _get(item.semantics, "name")) + if item.category == "QNP": + return ("quant", (_get(item.semantics, "quant"), _get(item.semantics, "restrictor"))) + return ("definite", _get(item.semantics, "lemma")) + + +def _object_of(item: "Item") -> tuple: + if item.category == "NAME": + return ("proper", _get(item.semantics, "name")) + if item.category == "QNP": + return ("quant", (_get(item.semantics, "quant"), _get(item.semantics, "restrictor"))) + return ("definite", _get(item.semantics, "lemma")) + + +def _sem(**kwargs) -> tuple: + return tuple(sorted(kwargs.items())) + + +def _get(semantics: tuple, key: str, default=None): + for name, value in semantics: + if name == key: + return value + return default + + +@dataclass +class ClauseProgram: + kind: str = "statement" # statement | question | conditional + quantifier: str = "proper" # proper | every | no | some | definite + restrictor: str | None = None + subject: str | None = None + predicate: dict[str, Any] = field(default_factory=dict) + obj: str | None = None + polarity: str = "+" + tense: str = "present" + modality: str = "none" + condition: "ClauseProgram | None" = None + conjuncts: "list[ClauseProgram] | None" = None + + def public(self) -> dict[str, Any]: + value = { + "kind": self.kind, "quantifier": self.quantifier, "restrictor": self.restrictor, + "subject": self.subject, "predicate": self.predicate, "object": self.obj, + "polarity": self.polarity, "tense": self.tense, "modality": self.modality, + } + if self.condition is not None: + value["condition"] = self.condition.public() + if self.conjuncts is not None: + value["conjuncts"] = [item.public() for item in self.conjuncts] + return value + + def digest(self) -> str: + return hashlib.sha256(json.dumps(self.public(), sort_keys=True).encode()).hexdigest() + + +class ConstructionKernel: + def __init__(self, *, compositional: bool = True): + self.compositional = compositional + self.memorized_signatures: set[tuple] = set() + + # ---------------- tokenization ---------------- + + @staticmethod + def _sentences(text: str) -> list[tuple[list[str], bool]]: + out = [] + for raw in re.split(r"(?<=[.?!])\s+", " ".join(str(text).split())): + raw = raw.strip() + if not raw: + continue + question = raw.endswith("?") + tokens = [t for t in re.findall(r"[A-Za-z][A-Za-z'-]*", raw) if t.casefold() not in {"then"}] + if tokens: + out.append((tokens, question)) + return out + + # ---------------- lexicon ---------------- + + @staticmethod + def _lexical(token: str, position: int) -> list[Item]: + low = token.casefold() + items: list[Item] = [] + span = (position, position + 1) + if low in QUANTIFIERS: + items.append(Item("Q", _sem(quant=QUANTIFIERS[low]), span)) + if low in ARTICLES: + items.append(Item("DET", _sem(article=low), span)) + if low in COPULAS: + tense, number = COPULAS[low] + items.append(Item("COP", _sem(tense=tense, number=number), span)) + if low in AUXILIARIES: + items.append(Item("AUX", _sem(tense=AUXILIARIES[low]), span)) + if low in NEGATIONS: + items.append(Item("NEG", _sem(), span)) + if low == "and": + items.append(Item("AND", _sem(), span)) + if low == "if": + items.append(Item("IF", _sem(), span)) + if token[0].isupper() and position > 0 and low not in QUANTIFIERS and low not in {"if"}: + items.append(Item("NAME", _sem(name=token), span)) + if token[0].isupper() and position == 0: + # sentence-initial capitalized token: could be a name or an ordinary word + items.append(Item("NAME", _sem(name=token), span)) + if low.isalpha(): + items.append(Item("N", _sem(lemma=_singular(low)), span)) + items.append(Item("ADJ", _sem(lemma=low), span)) + items.append(Item("VBASE", _sem(lemma=low), span)) + past = _lemma_from_past(low) + if past: + items.append(Item("VF", _sem(lemma=past, tense="past"), span)) + third = _lemma_from_3sg(low) + if third: + items.append(Item("VF", _sem(lemma=third, tense="present"), span)) + return items + + # ---------------- construction rules ---------------- + + @staticmethod + def _combine(left: Item, right: Item) -> list[Item]: + out: list[Item] = [] + span = (left.span[0], right.span[1]) + if left.category == "Q" and right.category == "N": + out.append(Item("QNP", _sem(quant=_get(left.semantics, "quant"), restrictor=_get(right.semantics, "lemma")), span)) + if left.category == "DET" and right.category == "N": + out.append(Item("DNP", _sem(article=_get(left.semantics, "article"), lemma=_get(right.semantics, "lemma")), span)) + if left.category == "NEG" and right.category == "ADJ": + out.append(Item("NEGADJ", _sem(lemma=_get(right.semantics, "lemma")), span)) + if left.category == "NEG" and right.category == "DNP": + out.append(Item("NEGDNP", _sem(lemma=_get(right.semantics, "lemma")), span)) + if left.category == "COP": + tense = _get(left.semantics, "tense") + if right.category == "ADJ": + out.append(Item("VP", _sem(kind="adjective", lemma=_get(right.semantics, "lemma"), polarity="+", tense=tense), span)) + if right.category == "NEGADJ": + out.append(Item("VP", _sem(kind="adjective", lemma=_get(right.semantics, "lemma"), polarity="-", tense=tense), span)) + if right.category == "DNP": + out.append(Item("VP", _sem(kind="nominal", lemma=_get(right.semantics, "lemma"), polarity="+", tense=tense), span)) + if right.category == "NEGDNP": + out.append(Item("VP", _sem(kind="nominal", lemma=_get(right.semantics, "lemma"), polarity="-", tense=tense), span)) + if left.category == "VF" and right.category in {"NAME", "QNP", "DNP"}: + out.append(Item("VP", _sem(kind="verb", lemma=_get(left.semantics, "lemma"), + obj=_object_of(right), polarity="+", tense=_get(left.semantics, "tense")), span)) + if left.category == "VBASE" and right.category in {"NAME", "QNP", "DNP"}: + out.append(Item("VPBASE", _sem(lemma=_get(left.semantics, "lemma"), obj=_object_of(right)), span)) + if left.category == "AUX" and right.category == "VPBASE": + out.append(Item("VP", _sem(kind="verb", lemma=_get(right.semantics, "lemma"), obj=_get(right.semantics, "obj"), + polarity="+", tense=_get(left.semantics, "tense")), span)) + if left.category == "AUX" and right.category == "NEG": + out.append(Item("AUXNEG", _sem(tense=_get(left.semantics, "tense")), span)) + if left.category == "AUXNEG" and right.category == "VPBASE": + out.append(Item("VP", _sem(kind="verb", lemma=_get(right.semantics, "lemma"), obj=_get(right.semantics, "obj"), + polarity="-", tense=_get(left.semantics, "tense")), span)) + if left.category in {"NAME", "QNP", "DNP"} and right.category == "VP": + out.append(Item("CL", _sem(subject=_subject_of(left), vp=right.semantics), span)) + if left.category == "AND" and right.category == "CL": + out.append(Item("ANDCL", right.semantics, span)) + if left.category == "CL" and right.category == "ANDCL": + out.append(Item("CLCONJ", _sem(first=left.semantics, second=right.semantics), span)) + if left.category == "IF" and right.category in {"CL", "CLCONJ"}: + out.append(Item("IFCL", right.semantics, span)) + if left.category == "IFCL" and right.category in {"CL", "CLCONJ"}: + out.append(Item("COND", _sem(condition=left.semantics, consequent=right.semantics), span)) + return out + + # ---------------- parsing ---------------- + + def _parse_tokens(self, tokens: list[str], question: bool) -> tuple[list[Item], tuple]: + if question and tokens and tokens[0].casefold() in (set(COPULAS) | set(AUXILIARIES)): + # subject-auxiliary inversion: [COP|AUX] NP REST -> NP [COP|AUX] REST, + # where NP spans two tokens when it opens with a determiner or quantifier + aux, rest = tokens[0], tokens[1:] + np_length = 2 if len(rest) >= 2 and rest[0].casefold() in (ARTICLES | set(QUANTIFIERS)) else 1 + tokens = [*rest[:np_length], aux, *rest[np_length:]] + n = len(tokens) + chart: dict[tuple, list[Item]] = {} + for i, token in enumerate(tokens): + chart[(i, i + 1)] = self._lexical(token, i) + for width in range(2, n + 1): + for start in range(0, n - width + 1): + end = start + width + cell: list[Item] = [] + seen: set[tuple] = set() + for middle in range(start + 1, end): + for left in chart.get((start, middle), ()): + for right in chart.get((middle, end), ()): + for item in self._combine(left, right): + key = (item.category, item.semantics) + if key not in seen: + seen.add(key) + cell.append(item) + chart[(start, end)] = cell + shape = tuple(t.casefold() if t.casefold() in CLOSED_CLASS else "W" for t in tokens) + return chart.get((0, n), []), shape + + # ---------------- item -> program ---------------- + + @staticmethod + def _clause_from(semantics: tuple, question: bool) -> ClauseProgram: + subject_kind, subject_value = _get(semantics, "subject") + vp = _get(semantics, "vp") + program = ClauseProgram(kind="question" if question else "statement") + if subject_kind == "proper": + program.subject = subject_value + elif subject_kind == "quant": + program.quantifier, program.restrictor = subject_value + else: + program.quantifier, program.subject = "definite", subject_value + program.predicate = {"kind": _get(vp, "kind"), "lemma": _get(vp, "lemma")} + obj = _get(vp, "obj") + if obj is not None: + program.obj = obj[1] if obj[0] == "proper" else obj[1] + program.polarity = _get(vp, "polarity") + program.tense = _get(vp, "tense") + return program + + def compile_sentence(self, tokens: list[str], question: bool) -> ClauseProgram | None: + items, shape = self._parse_tokens(tokens, question) + if not self.compositional: + if shape not in self.memorized_signatures: + return None + else: + self.memorized_signatures.add(shape) + for category in ("COND", "CLCONJ", "CL"): + for item in items: + if item.category != category: + continue + if category == "CL": + return self._clause_from(item.semantics, question) + if category == "CLCONJ": + first = self._clause_from(_get(item.semantics, "first"), question) + second = self._clause_from(_get(item.semantics, "second"), question) + program = ClauseProgram(kind="question" if question else "statement", conjuncts=[first, second]) + return program + condition = _get(item.semantics, "condition") + consequent = _get(item.semantics, "consequent") + program = ClauseProgram(kind="conditional") + program.condition = self._clause_from(condition, False) + consequent_program = self._clause_from(consequent, False) + program.predicate = consequent_program.predicate + program.subject = consequent_program.subject + program.quantifier = consequent_program.quantifier + program.restrictor = consequent_program.restrictor + program.obj = consequent_program.obj + program.polarity = consequent_program.polarity + program.tense = consequent_program.tense + program.modality = consequent_program.modality + return program + return None + + # ---------------- world + proof ---------------- + + def evaluate(self, text: str) -> dict[str, Any]: + """Compile a passage; statements build the temporary world; a final question is proved.""" + facts: list[ClauseProgram] = [] + rules: list[ClauseProgram] = [] + conditionals: list[ClauseProgram] = [] + question: ClauseProgram | None = None + unparsed: list[str] = [] + for tokens, is_question in self._sentences(text): + program = self.compile_sentence(tokens, is_question) + if program is None: + unparsed.append(" ".join(tokens)) + continue + targets = program.conjuncts if program.conjuncts else [program] + for target in targets: + if is_question: + question = target + elif target.kind == "conditional" or program.kind == "conditional": + conditionals.append(program if program.kind == "conditional" else target) + elif target.quantifier in {"every", "no"}: + rules.append(target) + else: + facts.append(target) + if unparsed: + return {"status": "clarification", "operation": "m13e2_unparsed_residue", + "response": "I could not compile: " + "; ".join(unparsed), + "unparsed": unparsed, "factual_release": False} + if question is None: + return {"status": "answered", "operation": "m13e2_world_update", + "response": f"Registered {len(facts)} facts, {len(rules)} quantified rules, " + f"{len(conditionals)} conditionals in the temporary world.", + "factual_release": False} + verdict, premises, rule_name = self._prove(question, facts, rules, conditionals) + answers = {"yes": "Yes", "no": "No", "unknown": "Unknown"} + conclusion = self._render(question, verdict) + return { + "status": "answered", "operation": "m13e2_compositional_premise_proof", + "response": f"{answers[verdict]}. {conclusion}", + "factual_release": False, "authority_class": "temporary_user_supplied_world", + "proof": {"premises": premises, "rule": rule_name, "conclusion": conclusion}, + "program": question.public(), "program_digest": question.digest(), + } + + def _prove(self, question: ClauseProgram, facts, rules, conditionals) -> tuple[str, list[str], str]: + # derive class memberships (kindOf closure over nominal facts and rules) + classes: dict[str, set[str]] = {} + for fact in facts: + if fact.predicate.get("kind") == "nominal" and fact.subject and fact.polarity == "+": + classes.setdefault(fact.subject, set()).add(fact.predicate["lemma"]) + changed = True + while changed: + changed = False + for rule in rules: + if rule.predicate.get("kind") != "nominal" or rule.quantifier != "every": + continue + for subject, kinds in classes.items(): + if rule.restrictor in kinds and rule.predicate["lemma"] not in kinds: + kinds.add(rule.predicate["lemma"]) + changed = True + # derived positive/negative property store: (subject, kind, lemma, obj, tense) -> polarity + derived: dict[tuple, tuple[str, str, str]] = {} + + def note(subject, predicate, obj, tense, polarity, premise, rule_name): + key = (subject, predicate["kind"], predicate["lemma"], obj, tense) + if key not in derived: + derived[key] = (polarity, premise, rule_name) + + for fact in facts: + note(fact.subject, fact.predicate, fact.obj, fact.tense, fact.polarity, + self._render(fact, "yes" if fact.polarity == "+" else "no"), "supplied_fact") + progressed = True + iterations = 0 + while progressed and iterations < 32: + progressed = False + iterations += 1 + for rule in rules: + for subject, kinds in classes.items(): + if rule.restrictor not in kinds: + continue + polarity = "-" if rule.quantifier == "no" else rule.polarity + key = (subject, rule.predicate["kind"], rule.predicate["lemma"], rule.obj, rule.tense) + if key not in derived: + derived[key] = (polarity, self._render(rule, "yes"), + "negative_universal_instantiation" if polarity == "-" else "universal_instantiation") + progressed = True + for conditional in conditionals: + antecedent = conditional.condition + key = (antecedent.subject, antecedent.predicate["kind"], antecedent.predicate["lemma"], + antecedent.obj, antecedent.tense) + held = derived.get(key) + if held and held[0] == antecedent.polarity: + consequent_key = (conditional.subject, conditional.predicate["kind"], + conditional.predicate["lemma"], conditional.obj, conditional.tense) + if consequent_key not in derived: + derived[consequent_key] = (conditional.polarity, + self._render(conditional, "yes"), "modus_ponens") + progressed = True + key = (question.subject, question.predicate["kind"], question.predicate["lemma"], question.obj, question.tense) + held = derived.get(key) + if held is None: + return "unknown", [], "not_derivable" + polarity, premise, rule_name = held + if polarity == question.polarity: + return "yes", [premise], rule_name + return "no", [premise], rule_name + + @staticmethod + def _render(program: ClauseProgram, verdict: str) -> str: + negative = (verdict == "no") ^ (program.polarity == "-") + subject = program.subject if program.subject else ( + {"every": "every", "no": "no", "some": "some"}.get(program.quantifier, "") + " " + str(program.restrictor)).strip() + predicate = program.predicate.get("lemma", "") + kind = program.predicate.get("kind") + if program.kind == "conditional" and program.condition is not None: + condition = ConstructionKernel._render(program.condition, "yes") + consequent = ConstructionKernel._render( + ClauseProgram(subject=program.subject, quantifier=program.quantifier, restrictor=program.restrictor, + predicate=program.predicate, obj=program.obj, polarity=program.polarity, + tense=program.tense), "yes") + return f"if {condition} then {consequent}" + copula = {"present": "is", "past": "was", "future": "will be"}[program.tense] + if kind == "verb": + verb = predicate if program.tense == "present" else predicate + aux = {"present": "does", "past": "did", "future": "will"}[program.tense] + if negative: + return f"{subject} {aux} not {verb} {program.obj}" + return f"{subject} {verb}{'s' if program.tense == 'present' and program.quantifier == 'proper' else ''} {program.obj}".strip() + if kind == "nominal": + article = "a " + return f"{subject} {copula}{' not' if negative else ''} {article}{predicate}" + return f"{subject} {copula}{' not' if negative else ''} {predicate}" + + +__all__ = ["ClauseProgram", "ConstructionKernel", "SCHEMA"] diff --git a/root/wander-train/stage5m13e2_construction_kernel_k4.py b/root/wander-train/stage5m13e2_construction_kernel_k4.py new file mode 100644 index 0000000000000000000000000000000000000000..52fe688121279d8affce3cd351b1b3bf3acb3bd6 --- /dev/null +++ b/root/wander-train/stage5m13e2_construction_kernel_k4.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""M13e2 K4: fold pass 2 — the banked residue gets homes. + +Donated by the registered debts of fold pass 1: + 7. object complements (made the mind affordable) — also resolves the recorded misparse hazard + 8. free relatives (what guts had labored over) — opaque definite description, semantics registered partial + 9. reduced relatives (the institution the species would build) — complex restrictor, semantics registered partial + 10. verb particles (labored over, kept back) + 11. past perfect (had labored) + 12. bare-plural subjects (guts labored) +Preference law: when a full-span object-complement reading exists, it outranks the +stacked-nominal reading (the pass-1 hazard becomes the dispreferred parse). +""" + +from __future__ import annotations + +import stage5m13e2_construction_kernel as k1 +from stage5m13e2_construction_kernel import CLOSED_CLASS, ClauseProgram, Item, _get, _sem, _subject_of +from stage5m13e2_construction_kernel_k2 import ConstructionKernelK2 +from stage5m13e2_construction_kernel_k3 import ConstructionKernelK3 + +SCHEMA = "oracle-stage5m13e2-construction-kernel-k4-v1" + +PARTICLES = {"over", "up", "back", "out", "off"} +k1.AUXILIARIES.setdefault("would", "past") +CLOSED_CLASS |= PARTICLES | {"would", "had"} +DROPPED_ADVERBS = {"ever"} + + +class ConstructionKernelK4(ConstructionKernelK3): + @staticmethod + def _sentences(text: str): + out = [] + for tokens, question in ConstructionKernelK3._sentences(text): + tokens = [t for t in tokens if t.casefold() not in DROPPED_ADVERBS] + if tokens: + out.append((tokens, question)) + return out + + def _lexical(self, token: str, position: int): + low = token.casefold() + span = (position, position + 1) + items = super()._lexical(token, position) + if low in PARTICLES: + items.append(Item("PRT", _sem(prt=low), span)) + if low == "had": + items.append(Item("AUXPERF", _sem(), span)) + if low in {"species", "series"}: # invariant nouns: singular form ends in -es + items = [Item("N", _sem(lemma=low), item.span) if item.category == "N" else item for item in items] + return items + + def _combine(self, left: Item, right: Item): + out = super()._combine(left, right) + span = (left.span[0], right.span[1]) + # 10. verb particles: VF/VBASE + PRT -> joined lemma + if left.category == "VF" and right.category == "PRT": + out.append(Item("VF", _sem(lemma=f"{_get(left.semantics, 'lemma')} {_get(right.semantics, 'prt')}", + tense=_get(left.semantics, "tense")), span)) + if left.category == "VBASE" and right.category == "PRT": + out.append(Item("VBASE", _sem(lemma=f"{_get(left.semantics, 'lemma')} {_get(right.semantics, 'prt')}"), span)) + # 11. past perfect: had + VF -> VF (tense past) + if left.category == "AUXPERF" and right.category == "VF": + out.append(Item("VF", _sem(lemma=_get(right.semantics, "lemma"), tense="past"), span)) + # 12. bare-plural subject: N + VF -> CL (intransitive gapped or plain) + if left.category == "N" and right.category == "VF": + out.append(Item("CL", _sem(subject=("definite", _get(left.semantics, "lemma")), + vp=_sem(kind="verb", lemma=_get(right.semantics, "lemma"), obj=None, + polarity="+", tense=_get(right.semantics, "tense"))), span)) + # 8. free relative: wh-NAME + gapped CL -> opaque definite description + if left.category == "NAME" and _get(left.semantics, "name") == "?x" and right.category == "CL": + vp = _get(right.semantics, "vp") + if _get(vp, "obj") is None and _get(vp, "kind") == "verb": + subject = _get(right.semantics, "subject") + description = f"what {subject[1]} {_get(vp, 'lemma')}".casefold() + out.append(Item("NAME", _sem(name=description), span)) + # 9. reduced relative: N + gapped definite-subject CL -> complex restrictor + if left.category == "N" and right.category == "CL": + vp = _get(right.semantics, "vp") + subject = _get(right.semantics, "subject") + if (_get(vp, "obj") is None and _get(vp, "kind") == "verb" + and isinstance(subject, tuple) and subject[0] == "definite"): + out.append(Item("N", _sem(lemma=f"{_get(left.semantics, 'lemma')} {subject[1]} {_get(vp, 'lemma')}"), span)) + # 7. object complement: transitive VP + ADJ -> VP(kind=objcomp) + if (left.category == "VP" and _get(left.semantics, "kind") == "verb" + and _get(left.semantics, "obj") is not None and right.category == "ADJ"): + out.append(Item("VP", _sem(kind="objcomp", lemma=_get(left.semantics, "lemma"), + obj=_get(left.semantics, "obj"), adj=_get(right.semantics, "lemma"), + polarity="+", tense=_get(left.semantics, "tense")), span)) + return out + + @staticmethod + def _clause_from(semantics: tuple, question: bool) -> ClauseProgram: + vp = _get(semantics, "vp") + if _get(vp, "kind") == "objcomp": + subject = _get(semantics, "subject") + obj = _get(vp, "obj") + obj_name = obj[1] + relation = ClauseProgram(kind="statement") + if subject[0] == "proper": + relation.subject = subject[1] + elif subject[0] == "quant": + relation.quantifier, relation.restrictor = subject[1] + else: + relation.quantifier, relation.subject = "definite", subject[1] + relation.predicate = {"kind": "verb", "lemma": _get(vp, "lemma")} + relation.obj = obj_name + relation.tense = _get(vp, "tense") + # resultative semantics: the caused property holds as a present result state + prop = ClauseProgram(kind="statement", quantifier="definite", subject=obj_name, + predicate={"kind": "adjective", "lemma": _get(vp, "adj")}, + tense="present") + return ClauseProgram(kind="question" if question else "statement", conjuncts=[relation, prop]) + return ConstructionKernelK3._clause_from(semantics, question) + + def compile_sentence(self, tokens, question: bool): + items, shape = self._parse_tokens(tokens, question) + if not self.compositional: + if shape not in self.memorized_signatures: + return None + else: + self.memorized_signatures.add(shape) + for item in items: + if item.category == "CLDISJ": + first = self._clause_from(_get(item.semantics, "first"), False) + second = self._clause_from(_get(item.semantics, "second"), False) + return ClauseProgram(kind="disjunction", conjuncts=[first, second]) + for category in ("COND", "CLCONJ"): + for item in items: + if item.category != category: + continue + if category == "CLCONJ": + first = self._clause_from(_get(item.semantics, "first"), question) + second = self._clause_from(_get(item.semantics, "second"), question) + return ClauseProgram(kind="question" if question else "statement", conjuncts=[first, second]) + condition = _get(item.semantics, "condition") + consequent = _get(item.semantics, "consequent") + program = ClauseProgram(kind="conditional") + program.condition = self._clause_from(condition, False) + consequent_program = self._clause_from(consequent, False) + program.predicate = consequent_program.predicate + program.subject = consequent_program.subject + program.quantifier = consequent_program.quantifier + program.restrictor = consequent_program.restrictor + program.obj = consequent_program.obj + program.polarity = consequent_program.polarity + program.tense = consequent_program.tense + program.modality = consequent_program.modality + return program + clauses = [item for item in items if item.category == "CL"] + # preference law: object-complement readings outrank stacked-nominal readings + clauses.sort(key=lambda item: 0 if _get(_get(item.semantics, "vp") or (), "kind") == "objcomp" else 1) + for item in clauses: + return self._clause_from(item.semantics, question) + return None + + +__all__ = ["ConstructionKernelK4", "SCHEMA"] diff --git a/root/wander-train/stage5m13e2_construction_kernel_k5.py b/root/wander-train/stage5m13e2_construction_kernel_k5.py new file mode 100644 index 0000000000000000000000000000000000000000..b4bde8a72585a2f9f5b564043cf73cca8587a928 --- /dev/null +++ b/root/wander-train/stage5m13e2_construction_kernel_k5.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""M13e2 K5: fold pass 3 — everyday conversational constructions. + + 13. there-existentials (There is a hammer on the bench.) + 14. possessives (Kip's hammer) + 15. prepositional locatives (in/on/at + "Where is X?") + 16. comparatives (taller than Sef) + 17. because-clauses (conjoined cause; causal link = registered debt) + 18. want/try + infinitive (attitude verbs with to-infinitive objects) + 19. plural copular agreement (are/were — verification pairs, no new rule) + 20. tag confirmations (", right?" stripped in question context) + 21. temporal when-clauses (conjoined; temporal link = registered debt) + 22. numeric determiners (two guards -> some+restrictor; exact count = registered debt) +""" + +from __future__ import annotations + +from stage5m13e2_construction_kernel import CLOSED_CLASS, COPULAS, ClauseProgram, Item, _get, _sem +from stage5m13e2_construction_kernel_k4 import ConstructionKernelK4 + +SCHEMA = "oracle-stage5m13e2-construction-kernel-k5-v1" + +PREPOSITIONS = {"in", "on", "at"} +NUMBER_DETERMINERS = {"two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10} +CLOSED_CLASS |= PREPOSITIONS | set(NUMBER_DETERMINERS) | {"there", "than", "because", "to", "when", "locwh"} + + +def _comparative_base(word: str) -> str: + stem = word[:-2] + # de-double gemination (bigger -> big) but keep legitimate doubles (taller, lesser) + if len(stem) >= 2 and stem[-1] == stem[-2] and stem[-1] not in "lse": + stem = stem[:-1] + return stem + + +class ConstructionKernelK5(ConstructionKernelK4): + @staticmethod + def _sentences(text: str): + out = [] + for tokens, question in ConstructionKernelK4._sentences(text): + if question and tokens and tokens[-1].casefold() == "right": + tokens = tokens[:-1] # tag confirmation: ", right?" + if tokens: + out.append((tokens, question)) + return out + + def _lexical(self, token: str, position: int): + low = token.casefold() + span = (position, position + 1) + items = super()._lexical(token, position) + if low in PREPOSITIONS: + items.append(Item("PREP", _sem(prep=low), span)) + if low == "there": + items.append(Item("THERE", _sem(), span)) + if low == "than": + items.append(Item("THAN", _sem(), span)) + if low == "because": + items.append(Item("BECAUSE", _sem(), span)) + if low == "when": + items.append(Item("WHEN", _sem(), span)) + if low == "to": + items.append(Item("TO", _sem(), span)) + if low in NUMBER_DETERMINERS: + items.append(Item("NUMDET", _sem(count=NUMBER_DETERMINERS[low]), span)) + if low == "locwh": # injected by the where-question transform + items.append(Item("PPLOC", _sem(prep="?", place="?x"), span)) + if low.endswith("'s") and len(low) > 2: + items = [item for item in items if item.category != "NAME"] + items.append(Item("POSS", _sem(owner=low[:-2]), span)) + return items + + def _parse_tokens(self, tokens, question: bool): + # where-question transform: Where is NP? -> NP is locwh + if question and tokens and tokens[0].casefold() == "where" and len(tokens) >= 3 \ + and tokens[1].casefold() in COPULAS: + tokens = [*tokens[2:], tokens[1], "locwh"] + return super()._parse_tokens(tokens, False) + return super()._parse_tokens(tokens, question) + + def _combine(self, left: Item, right: Item): + out = super()._combine(left, right) + span = (left.span[0], right.span[1]) + # 15. locative PP: PREP + (DNP|NAME|N) -> PPLOC + if left.category == "PREP" and right.category in {"DNP", "NAME", "N"}: + place = _get(right.semantics, "name") or _get(right.semantics, "lemma") + out.append(Item("PPLOC", _sem(prep=_get(left.semantics, "prep"), place=str(place).casefold()), span)) + if left.category == "COP" and right.category == "PPLOC": + place = _get(right.semantics, "place") + out.append(Item("VP", _sem(kind="location", lemma=_get(right.semantics, "prep"), + obj=("proper", "?x") if place == "?x" else ("definite", place), + polarity="+", tense=_get(left.semantics, "tense")), span)) + # 13. existentials: (DNP|N) + PPLOC -> NPLOC ; COP + NPLOC -> VPEX ; THERE + VPEX -> CL + if left.category in {"DNP", "N"} and right.category == "PPLOC": + out.append(Item("NPLOC", _sem(entity=_get(left.semantics, "lemma"), + prep=_get(right.semantics, "prep"), + place=_get(right.semantics, "place")), span)) + if left.category == "COP" and right.category == "NPLOC": + out.append(Item("VPEX", _sem(kind="location", lemma=_get(right.semantics, "prep"), + obj=("definite", _get(right.semantics, "place")), + entity=_get(right.semantics, "entity"), + polarity="+", tense=_get(left.semantics, "tense")), span)) + if left.category == "THERE" and right.category == "VPEX": + vp = _sem(kind="location", lemma=_get(right.semantics, "lemma"), obj=_get(right.semantics, "obj"), + polarity="+", tense=_get(right.semantics, "tense")) + out.append(Item("CL", _sem(subject=("quant", ("some", _get(right.semantics, "entity"))), vp=vp), span)) + # 14. possessives: POSS + N -> DNP (possessives are definite descriptions) + if left.category == "POSS" and right.category == "N": + out.append(Item("DNP", _sem(article="poss", + lemma=f"{_get(left.semantics, 'owner')}'s {_get(right.semantics, 'lemma')}"), span)) + # 16. comparatives: THAN + (NAME|DNP) -> THANP ; ADJ + THANP -> ADJCOMP ; COP (+NEG) + ADJCOMP -> VP + if left.category == "THAN" and right.category in {"NAME", "DNP"}: + std = _get(right.semantics, "name") or _get(right.semantics, "lemma") + out.append(Item("THANP", _sem(standard=str(std)), span)) + if left.category == "ADJ" and right.category == "THANP" and str(_get(left.semantics, "lemma")).endswith("er"): + out.append(Item("ADJCOMP", _sem(dimension=_comparative_base(_get(left.semantics, "lemma")), + standard=_get(right.semantics, "standard")), span)) + if left.category == "NEG" and right.category == "ADJCOMP": + out.append(Item("NEGADJCOMP", right.semantics, span)) + if left.category == "COP" and right.category == "ADJCOMP": + out.append(Item("VP", _sem(kind="comparative", lemma=_get(right.semantics, "dimension"), + obj=("proper", _get(right.semantics, "standard")), + polarity="+", tense=_get(left.semantics, "tense")), span)) + if left.category == "COP" and right.category == "NEGADJCOMP": + out.append(Item("VP", _sem(kind="comparative", lemma=_get(right.semantics, "dimension"), + obj=("proper", _get(right.semantics, "standard")), + polarity="-", tense=_get(left.semantics, "tense")), span)) + # 17/21. because/when clause pairs + if left.category == "BECAUSE" and right.category == "CL": + out.append(Item("BECCL", right.semantics, span)) + if left.category == "CL" and right.category == "BECCL": + out.append(Item("CLCAUSE", _sem(main=left.semantics, cause=right.semantics), span)) + if left.category == "WHEN" and right.category == "CL": + out.append(Item("WHENCL", right.semantics, span)) + if left.category == "WHENCL" and right.category == "CL": + out.append(Item("CLTEMP", _sem(when=left.semantics, main=right.semantics), span)) + # 18. infinitives: TO + VBASE -> INF ; VF/VBASE + INF -> attitude VP + if left.category == "TO" and right.category == "VBASE": + out.append(Item("INF", _sem(verb=_get(right.semantics, "lemma")), span)) + if left.category == "VF" and right.category == "INF": + out.append(Item("VP", _sem(kind="attitude", lemma=_get(left.semantics, "lemma"), + obj=("inf", f"to {_get(right.semantics, 'verb')}"), + polarity="+", tense=_get(left.semantics, "tense")), span)) + if left.category == "VBASE" and right.category == "INF": + out.append(Item("VPBASE", _sem(lemma=_get(left.semantics, "lemma"), + obj=("inf", f"to {_get(right.semantics, 'verb')}")), span)) + # 22. numeric determiners: NUMDET + N -> QNP (count = registered debt) + if left.category == "NUMDET" and right.category == "N": + out.append(Item("QNP", _sem(quant="some", restrictor=_get(right.semantics, "lemma"), + count=_get(left.semantics, "count")), span)) + return out + + def compile_sentence(self, tokens, question: bool): + items, shape = self._parse_tokens(tokens, question) + if not self.compositional: + if shape not in self.memorized_signatures: + return None + else: + self.memorized_signatures.add(shape) + for item in items: + if item.category in {"CLCAUSE", "CLTEMP"}: + if item.category == "CLCAUSE": + first = self._clause_from(_get(item.semantics, "main"), False) + second = self._clause_from(_get(item.semantics, "cause"), False) + else: + first = self._clause_from(_get(item.semantics, "when"), False) + second = self._clause_from(_get(item.semantics, "main"), False) + return ClauseProgram(kind="question" if question else "statement", conjuncts=[first, second]) + return super().compile_sentence(tokens, question) + + def _wh_matches(self, question: ClauseProgram, derived): + matches = super()._wh_matches(question, derived) + if matches or question.predicate.get("kind") != "location" or question.predicate.get("lemma") != "?": + return matches + # where-question: match any locative lemma for the same subject + for (subject, kind, lemma, obj, tense, _modality), (polarity, _p, _r) in derived.items(): + if kind == "location" and polarity == "+" and subject == question.subject and obj not in (None, "?x"): + matches.append(f"{lemma} the {obj}") + return matches + + +__all__ = ["ConstructionKernelK5", "SCHEMA"] diff --git a/root/wander-train/stage5m13e2_construction_kernel_k7b.py b/root/wander-train/stage5m13e2_construction_kernel_k7b.py new file mode 100644 index 0000000000000000000000000000000000000000..65ad60561b8d8f30f0807ca3f972dad593de3448 --- /dev/null +++ b/root/wander-train/stage5m13e2_construction_kernel_k7b.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""K7b — recursive composition successor to K7 (frozen; not edited). + +The K7 withheld one-shot failed 1/10 (stage5m13e6_k7_withheld_products_report_v1.json, +preserved): builders were first-match-wins and sub-clauses went only to the +plain K6 clause parser, so families never composed. K7b makes decomposition +RECURSIVE: every sub-clause request routes back through compile_text (with a +depth guard), so a connective can hold a relative, an appositive can hold an +attitude report, a measure can hold a possessive chain. Also: connective moved +ahead of existential (the F6 bypass that broke ablation attribution), a +broadened cleft (non-capitalized focus NPs), and the parenthetical attitude +construction ("X, Ferra said, Y").""" + +from __future__ import annotations + +import re + +from stage5m13e2_construction_kernel import ClauseProgram +from stage5m13e2_construction_kernel_k7 import ( + ConstructionKernelK7, LinkedProgram, PRE_ADVERBS, +) + +PARENTHETICAL_ATTITUDE = re.compile( + r"^(.+?), ([A-Z]\w+) (said|says|believed|believes|claimed|claims), (.+?)\.?$") +CLEFT_ANY = re.compile(r"^It (was|is) (.+?) (who|that) (.+?)\.?$") +_MAX_DEPTH = 6 + + +class ConstructionKernelK7b(ConstructionKernelK7): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._depth = 0 + + # ── recursion: sub-clause requests re-enter the full builder stack ─── + def _clause(self, text: str): # may now return LinkedProgram + if self._depth >= _MAX_DEPTH: + return super()._clause(text) + self._depth += 1 + try: + return self._compile_inner(text) + finally: + self._depth -= 1 + + NUMERALS = {"one", "two", "three", "four", "five", "six", "seven", + "eight", "nine", "ten"} + + def _plain_clause(self, text: str) -> ClauseProgram | None: + program = super()._clause(text) + if program is not None: + return program + # elliptical numeral object ("the plans demanded four" = four [units], + # anaphoric to a measure in the host clause): parse without the numeral + # and stash it, symmetric with the adverbial lift. + words = text.strip().rstrip(".").split() + if len(words) >= 3 and words[-1].lower() in self.NUMERALS: + program = super()._clause(" ".join(words[:-1])) + if program is not None: + program.predicate = {**program.predicate, + "k7_numeral_object": words[-1].lower()} + return program + + # ── new/overridden builders ────────────────────────────────────────── + def _parenthetical_attitude(self, text: str) -> LinkedProgram | None: + match = PARENTHETICAL_ATTITUDE.match(text.strip()) + if not match: + return None + first, holder, verb, rest = match.groups() + content = self._clause(f"{first} {rest}") + if content is None: + return None + lemma = {"said": "say", "says": "say", "believed": "believe", "believes": "believe", + "claimed": "claim", "claims": "claim"}[verb] + tense = "past" if verb in ("said", "believed", "claimed") else "present" + return LinkedProgram("attitude", content, None, + meta={"holder": holder, "verb_lemma": lemma, "tense": tense, + "world": "reported", "parenthetical": True, + "split_at": first}) + + def _cleft(self, text: str) -> LinkedProgram | None: + match = CLEFT_ANY.match(text.strip()) + if not match: + return None + copula, focus, relativizer, rest = match.groups() + if "," in focus or len(focus.split()) > 5: + return None + base = self._clause(f"{focus} {rest}") + if base is None: + return None + return LinkedProgram("cleft_focus", base, None, + meta={"focus": focus, "copula": copula, "relativizer": relativizer}) + + # measure/phrasal tense assignment must tolerate a linked base + def _measure(self, text: str): + program = super()._measure(text) + return program + + # ── entry with recursive order (connective BEFORE existential) ─────── + def _compile_inner(self, text: str): + stripped = text.strip() + if not stripped: + return None + stripped = stripped[0].upper() + stripped[1:] + pre_adverb = None + words = stripped.rstrip(".").split() + for adverb in PRE_ADVERBS: + if adverb in words[1:-1]: + index = words.index(adverb) + stripped = " ".join(words[:index] + words[index + 1:]) + "." + pre_adverb = adverb + break + disabled = getattr(self, "disabled_families", frozenset()) + family_builders = ( + ("F9", self._parenthetical_attitude), ("F8", self._cleft), + ("F9", self._attitude), ("F10", self._neg_scope), + ("F3", self._appositive), ("F13", self._verbal_subject), + ("F1", self._connective), ("F4", self._fronted_adverbial), + ("F5", self._coordination_ellipsis), ("F7", self._measure), + ("F12", self._phrasal), ("F14", self._passive_agent), + ("F11", self._possessive_chain), ("F2", self._relative), + ("F6", self._existential)) + for family, builder in family_builders: + if family in disabled: + continue + program = builder(stripped) + if program is not None: + if pre_adverb: + target = program.main if isinstance(program, LinkedProgram) else program + if isinstance(target, ClauseProgram): + target.predicate = {**target.predicate, "k7_pre_adverb": pre_adverb} + return program + clause = self._plain_clause(stripped) + if clause is not None and pre_adverb: + clause.predicate = {**clause.predicate, "k7_pre_adverb": pre_adverb} + return clause + + def compile_text(self, text: str): + self._depth = 1 + try: + return self._compile_inner(text) + finally: + self._depth = 0 diff --git a/root/wander-train/stage5m13e4_combinators.py b/root/wander-train/stage5m13e4_combinators.py new file mode 100644 index 0000000000000000000000000000000000000000..fb84fa5b75eb21fa0c10c9628ecec2e4381b60e1 --- /dev/null +++ b/root/wander-train/stage5m13e4_combinators.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""M13e4 combinator/translation layer: ClauseProgram -> typed lambda IR + IR proof world. + +Staged migration per spec: the K6 chart remains the parser; its ClauseProgram output +translates into beta-normalized, alpha-invariant, world-scoped IR terms. Inference +runs ON TERMS: universal instantiation IS beta-application of a stored scope lambda. +""" + +from __future__ import annotations + +from typing import Any + +from stage5m13e2_construction_kernel import ClauseProgram +from stage5m13e4_oracle_logical_ir import ( + App, Const, E, EpistemicStore, Fun, GQ, Lam, T, Term, Var, digest, infer, normalize, pred, +) + +SCHEMA = "oracle-stage5m13e4-combinators-v1" + +ISA = Const("isa", pred(E, E, T)) +NOT = Const("not", Fun(T, T)) +TENSE = {name: Const(f"tense:{name}", Fun(T, T)) for name in ("present", "past", "future")} +MODAL = {name: Const(f"modal:{name}", Fun(T, T)) for name in ("must", "may", "can", "could")} + +BINARY_KINDS = {"verb", "location", "comparative", "recipient", "attitude", "passive"} + + +def entity(name: Any) -> Const: + return Const(f"ent:{name}", E) + + +def class_entity(lemma: Any) -> Const: + return Const(f"class:{lemma}", E) + + +def _atom(program: ClauseProgram, subject_term: Term) -> Term: + kind = program.predicate.get("kind") + lemma = program.predicate.get("lemma") + if kind == "nominal": + return App(App(ISA, subject_term), class_entity(lemma)) + if kind in BINARY_KINDS and program.obj is not None: + relation = Const(f"{kind}:{lemma}", pred(E, E, T)) + return App(App(relation, subject_term), entity(program.obj)) + predicate = Const(f"{kind}:{lemma}", pred(E, T)) + return App(predicate, subject_term) + + +def _wrap(program: ClauseProgram, atom: Term) -> Term: + term = atom + if program.modality and program.modality != "none": + term = App(MODAL.get(program.modality, Const(f"modal:{program.modality}", Fun(T, T))), term) + term = App(TENSE.get(program.tense, TENSE["present"]), term) + return term + + +def core_term(program: ClauseProgram, subject_term: Term) -> Term: + """Polarity is carried OUTSIDE the core (mirrors the kernel's key/polarity split).""" + return normalize(_wrap(program, _atom(program, subject_term))) + + +def subject_entity(program: ClauseProgram) -> Term: + if program.subject is not None: + prefix = "ent:" if program.quantifier == "proper" else "ent:def:" + return Const(prefix + str(program.subject), E) + return Const(f"ent:sk:{program.restrictor}", E) # existential skolem + + +class IRProofWorld: + """Ground store + lambda rules; the four inference laws run on normalized terms.""" + + def __init__(self, world: str = "user_temporary_world"): + self.world = world + self.store = EpistemicStore() + self.polarity: dict[str, str] = {} + self.rules: list[dict] = [] + self.conditionals: list[dict] = [] + self.disjunctions: list[dict] = [] + self.entities: set[str] = set() + self._cores: list[Term] = [] + + # ---- assertion ---- + + def _assert_core(self, core: Term, polarity: str, provenance: str): + self.store.assert_(self.world, core, polarity == "+", provenance) + self._cores.append(core) + key = digest(core) + held = self.polarity.get(key) + self.polarity[key] = polarity if held in (None, polarity) else "!" + + def assert_program(self, program: ClauseProgram, provenance: str = "supplied"): + if program.conjuncts and program.kind == "disjunction": + first, second = program.conjuncts + self.disjunctions.append({ + "a": core_term(first, subject_entity(first)), "pa": first.polarity, + "b": core_term(second, subject_entity(second)), "pb": second.polarity}) + for side in (first, second): + self.entities.add(digest(subject_entity(side))) + return + if program.kind == "conditional" and program.condition is not None: + antecedent = program.condition + self.conditionals.append({ + "ant": core_term(antecedent, subject_entity(antecedent)), "ant_pol": antecedent.polarity, + "cons_program": program}) + return + if program.conjuncts: + for part in program.conjuncts: + self.assert_program(part, provenance) + return + if program.quantifier in {"every", "no"}: + x = Var("x", E) + scope = Lam(x, _wrap(program, _atom(program, x))) + infer(scope) + self.rules.append({"restrictor": str(program.restrictor), "scope": scope, + "polarity": "-" if program.quantifier == "no" else program.polarity}) + return + subject = subject_entity(program) + self.entities.add(digest(subject)) + if program.quantifier == "some" and program.restrictor: + self._assert_core(normalize(App(TENSE[program.tense], App(App(ISA, subject), class_entity(program.restrictor)))), + "+", provenance) + self._assert_core(core_term(program, subject), program.polarity, provenance) + + # ---- inference: instantiation IS beta-application ---- + + def saturate(self, budget: int = 64): + constants: dict[str, Term] = {} + + def collect(term: Term): + if isinstance(term, Const) and term.ctype == E and term.name.startswith("ent:"): + constants[term.name] = term + elif isinstance(term, App): + collect(term.fn) + collect(term.arg) + elif isinstance(term, Lam): + collect(term.body) + + for record in self.rules: + collect(record["scope"]) + for record in self.conditionals: + collect(record["ant"]) + for record in self.disjunctions: + collect(record["a"]) + collect(record["b"]) + for core in self._cores: + collect(core) + + progressed = True + iterations = 0 + while progressed and iterations < budget: + progressed = False + iterations += 1 + for rule in self.rules: + member_class = class_entity(rule["restrictor"]) + for name, const in list(constants.items()): + membership = None + for tense_const in TENSE.values(): + candidate = normalize(App(tense_const, App(App(ISA, const), member_class))) + if self.polarity.get(digest(candidate)) == "+": + membership = True + break + if not membership: + continue + derived = normalize(App(rule["scope"], const)) + key = digest(derived) + if key not in self.polarity: + self._assert_core(derived, rule["polarity"], f"instantiation:{rule['restrictor']}") + collect(derived) + progressed = True + for conditional in self.conditionals: + if self.polarity.get(digest(conditional["ant"])) == conditional["ant_pol"]: + consequent = conditional["cons_program"] + core = core_term(consequent, subject_entity(consequent)) + if digest(core) not in self.polarity: + self._assert_core(core, consequent.polarity, "modus_ponens") + collect(core) + progressed = True + for disjunction in self.disjunctions: + for this_core, this_pol, other_core, other_pol in ( + (disjunction["a"], disjunction["pa"], disjunction["b"], disjunction["pb"]), + (disjunction["b"], disjunction["pb"], disjunction["a"], disjunction["pa"])): + held = self.polarity.get(digest(this_core)) + if held is not None and held != this_pol and digest(other_core) not in self.polarity: + self._assert_core(other_core, other_pol, "disjunctive_syllogism") + collect(other_core) + progressed = True + + def ask(self, program: ClauseProgram) -> str: + core = core_term(program, subject_entity(program)) + held = self.polarity.get(digest(core)) + if held is None or held == "!": + return "unknown" if held is None else ("yes" if program.polarity == "+" else "no") + return "yes" if held == program.polarity else "no" + + +class IRPipeline: + """kernel parse -> translate -> IR world -> verdict (yes/no/unknown path).""" + + def __init__(self, kernel): + self.kernel = kernel + + def answer(self, passage: str) -> str: + world = IRProofWorld() + last_proper = None + from stage5m13e2_construction_kernel_k2 import PRONOUNS + question_program = None + for tokens, is_question in self.kernel._sentences(passage): + lows = [t.casefold() for t in tokens] + if lows[:3] == ["no", "i", "meant"] and len(tokens) >= 4: + # repair handled by the kernel path; replay via kernel world instead + return "UNSUPPORTED_REPAIR" + tokens = [last_proper if (t.casefold() in PRONOUNS and last_proper) else t for t in tokens] + program = self.kernel.compile_sentence(tokens, is_question) + if program is None: + continue + if is_question: + question_program = program if not program.conjuncts else program.conjuncts[-1] + else: + world.assert_program(program) + target = program.conjuncts[0] if program.conjuncts and program.kind != "disjunction" else program + if target.subject and str(target.subject)[:1].isupper(): + last_proper = str(target.subject) + if question_program is None: + return "none" + world.saturate() + return world.ask(question_program) + + +def translate(program: ClauseProgram) -> Term: + """Full typed translation of one program (for the translation gate).""" + if program.conjuncts: + AND = Const("and", Fun(T, Fun(T, T))) + term = translate(program.conjuncts[0]) + for part in program.conjuncts[1:]: + term = App(App(AND, term), translate(part)) + return normalize(term) + if program.kind == "conditional" and program.condition is not None: + IF = Const("if", Fun(T, Fun(T, T))) + consequent = ClauseProgram(**{**program.__dict__, "kind": "statement", "condition": None, "conjuncts": None}) + return normalize(App(App(IF, translate(program.condition)), translate(consequent))) + if program.quantifier in {"every", "no", "some"} and program.restrictor: + quant = Const(program.quantifier, GQ) + x = Var("x", E) + restrictor = Lam(x, App(App(ISA, x), class_entity(program.restrictor))) + scope = Lam(x, _wrap(program, _atom(program, x))) + term = App(App(quant, restrictor), scope) + return normalize(term if program.polarity == "+" else App(NOT, term)) + core = _wrap(program, _atom(program, subject_entity(program))) + return normalize(core if program.polarity == "+" else App(NOT, core)) + + +__all__ = ["IRPipeline", "IRProofWorld", "SCHEMA", "core_term", "translate"] diff --git a/root/wander-train/stage5m13e4_oracle_logical_ir.py b/root/wander-train/stage5m13e4_oracle_logical_ir.py new file mode 100644 index 0000000000000000000000000000000000000000..1bc1db76b9ea5b83b9ba9da6fadae725701827d6 --- /dev/null +++ b/root/wander-train/stage5m13e4_oracle_logical_ir.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +"""M13e4 core: the Oracle Logical IR. + +Simply-typed lambda calculus over the registered base types, with deterministic +capture-avoiding beta normalization, alpha-invariant canonical digests, and a +six-valued epistemic store (Belnap TRUE/FALSE/BOTH/UNKNOWN plus HYPOTHESIS and +UNSUPPORTED) whose propositions carry exactly one world scope and whose +world/authority boundaries are crossable only through registered operators. + +No transformer, no embedding, no gradient, no external model. Pure structure. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import Any, Union + +SCHEMA = "oracle-stage5m13e4-logical-ir-v1" + +# ---------------------------------------------------------------- types + +BASE_TYPES = { + "Entity", "Truth", "World", "Event", "Time", "Location", "Quantity", + "Evidence", "Claim", "Speaker", "Goal", "Action", "Specialist", + "Objective", "Authority", +} + + +@dataclass(frozen=True) +class Base: + name: str + + def __post_init__(self): + if self.name not in BASE_TYPES: + raise TypeError(f"unknown base type: {self.name}") + + def __repr__(self): + return self.name + + +@dataclass(frozen=True) +class Fun: + domain: "Type" + codomain: "Type" + + def __repr__(self): + return f"({self.domain} -> {self.codomain})" + + +Type = Union[Base, Fun] + +E, T, W = Base("Entity"), Base("Truth"), Base("World") +EV, TM = Base("Event"), Base("Time") + + +def pred(*args: Type) -> Type: + """pred(A, B, ..., R) builds the curried type A -> B -> ... -> R.""" + result = args[-1] + for domain in reversed(args[:-1]): + result = Fun(domain, result) + return result + + +GQ = Fun(Fun(E, T), Fun(Fun(E, T), T)) # generalized quantifier + +# ---------------------------------------------------------------- terms + + +@dataclass(frozen=True) +class Var: + name: str + vtype: Type + + +@dataclass(frozen=True) +class Const: + name: str + ctype: Type + + +@dataclass(frozen=True) +class Lam: + param: Var + body: "Term" + + +@dataclass(frozen=True) +class App: + fn: "Term" + arg: "Term" + + +Term = Union[Var, Const, Lam, App] + + +class TypeError_(Exception): + pass + + +def infer(term: Term, env: dict[str, Type] | None = None) -> Type: + env = env or {} + if isinstance(term, Var): + bound = env.get(term.name) + if bound is not None and bound != term.vtype: + raise TypeError_(f"variable {term.name} annotated {term.vtype} but bound {bound}") + return term.vtype + if isinstance(term, Const): + return term.ctype + if isinstance(term, Lam): + inner = dict(env) + inner[term.param.name] = term.param.vtype + return Fun(term.param.vtype, infer(term.body, inner)) + if isinstance(term, App): + fn_type = infer(term.fn, env) + arg_type = infer(term.arg, env) + if not isinstance(fn_type, Fun): + raise TypeError_(f"applying non-function of type {fn_type}") + if fn_type.domain != arg_type: + raise TypeError_(f"domain {fn_type.domain} != argument {arg_type}") + return fn_type.codomain + raise TypeError_(f"unknown term {term!r}") + + +def free_variables(term: Term) -> set[str]: + if isinstance(term, Var): + return {term.name} + if isinstance(term, Const): + return set() + if isinstance(term, Lam): + return free_variables(term.body) - {term.param.name} + return free_variables(term.fn) | free_variables(term.arg) + + +class _Fresh: + def __init__(self): + self.counter = 0 + + def name(self, base: str) -> str: + self.counter += 1 + return f"{base}#{self.counter}" + + +def _substitute(term: Term, name: str, value: Term, fresh: _Fresh) -> Term: + if isinstance(term, Var): + return value if term.name == name else term + if isinstance(term, Const): + return term + if isinstance(term, App): + return App(_substitute(term.fn, name, value, fresh), + _substitute(term.arg, name, value, fresh)) + if isinstance(term, Lam): + if term.param.name == name: + return term # shadowed + if term.param.name in free_variables(value): + renamed = Var(fresh.name(term.param.name.split("#")[0]), term.param.vtype) + body = _substitute(term.body, term.param.name, renamed, fresh) + return Lam(renamed, _substitute(body, name, value, fresh)) + return Lam(term.param, _substitute(term.body, name, value, fresh)) + raise TypeError_(f"unknown term {term!r}") + + +def _step(term: Term, fresh: _Fresh) -> Term | None: + """One leftmost-outermost beta step, or None at normal form.""" + if isinstance(term, App): + if isinstance(term.fn, Lam): + return _substitute(term.fn.body, term.fn.param.name, term.arg, fresh) + reduced = _step(term.fn, fresh) + if reduced is not None: + return App(reduced, term.arg) + reduced = _step(term.arg, fresh) + if reduced is not None: + return App(term.fn, reduced) + return None + if isinstance(term, Lam): + reduced = _step(term.body, fresh) + if reduced is not None: + return Lam(term.param, reduced) + return None + return None + + +def normalize(term: Term, budget: int = 10_000) -> Term: + """Deterministic beta normalization; type-check first (typing guarantees termination).""" + infer(term) + fresh = _Fresh() + for _ in range(budget): + reduced = _step(term, fresh) + if reduced is None: + return term + term = reduced + raise TypeError_("normalization budget exceeded (ill-typed self-application?)") + + +# ------------------------------------------------- canonical form + digest + + +def _type_obj(t: Type) -> Any: + if isinstance(t, Base): + return t.name + return [_type_obj(t.domain), _type_obj(t.codomain)] + + +def _canonical(term: Term, binders: tuple[str, ...]) -> Any: + """de Bruijn form: alpha-equivalent terms serialize identically.""" + if isinstance(term, Var): + if term.name in binders: + return ["b", len(binders) - 1 - binders.index(term.name), _type_obj(term.vtype)] + return ["f", term.name, _type_obj(term.vtype)] + if isinstance(term, Const): + return ["c", term.name, _type_obj(term.ctype)] + if isinstance(term, Lam): + return ["l", _type_obj(term.param.vtype), _canonical(term.body, binders + (term.param.name,))] + return ["a", _canonical(term.fn, binders), _canonical(term.arg, binders)] + + +def canonical(term: Term) -> str: + return json.dumps(_canonical(term, ()), sort_keys=True, separators=(",", ":")) + + +def digest(term: Term) -> str: + return hashlib.sha256(canonical(term).encode()).hexdigest() + + +def alpha_equal(left: Term, right: Term) -> bool: + return canonical(left) == canonical(right) + + +# --------------------------------------------- epistemic store with worlds + +STATUSES = ("TRUE", "FALSE", "BOTH", "UNKNOWN", "HYPOTHESIS", "UNSUPPORTED") + +WORLD_SCOPES = ( + "authenticated_fact", "user_temporary_world", "hypothetical_world", + "counterfactual_world", "reported_belief", "fictional_world", + "episodic_tool_observation", +) + +# world-crossing operators: (source_scope, operator) -> target_scope +REGISTERED_CROSSINGS = { + ("user_temporary_world", "report_as_belief"): "reported_belief", + ("authenticated_fact", "report_as_belief"): "reported_belief", + ("episodic_tool_observation", "adopt_observation"): "user_temporary_world", + ("hypothetical_world", "discharge_hypothesis"): "user_temporary_world", +} + + +class WorldBoundaryError(Exception): + pass + + +class EpistemicStore: + """Six-valued store; BOTH preserves all conflicting supports (paraconsistent).""" + + def __init__(self): + self.entries: dict[tuple[str, str], dict[str, Any]] = {} + + @staticmethod + def _key(world: str, proposition: Term) -> tuple[str, str]: + if world not in WORLD_SCOPES: + raise WorldBoundaryError(f"unknown world scope: {world}") + if infer(proposition) != T: + raise TypeError_("only Truth-typed terms are propositions") + return (world, digest(normalize(proposition))) + + def assert_(self, world: str, proposition: Term, polarity: bool, provenance: str) -> str: + key = self._key(world, proposition) + entry = self.entries.setdefault(key, {"status": "UNKNOWN", "supports": []}) + entry["supports"].append({"polarity": polarity, "provenance": provenance}) + polarities = {support["polarity"] for support in entry["supports"]} + if polarities == {True}: + entry["status"] = "TRUE" + elif polarities == {False}: + entry["status"] = "FALSE" + else: + entry["status"] = "BOTH" # all conflicting supports retained, no explosion + return entry["status"] + + def hypothesize(self, world: str, proposition: Term, provenance: str) -> str: + key = self._key(world, proposition) + entry = self.entries.setdefault(key, {"status": "UNKNOWN", "supports": []}) + if entry["status"] == "UNKNOWN": + entry["status"] = "HYPOTHESIS" + entry["supports"].append({"polarity": None, "provenance": provenance, "hypothesis": True}) + return entry["status"] + + def mark_unsupported(self, world: str, proposition: Term, provenance: str) -> str: + key = self._key(world, proposition) + entry = self.entries.setdefault(key, {"status": "UNKNOWN", "supports": []}) + if entry["status"] in ("UNKNOWN", "HYPOTHESIS"): + entry["status"] = "UNSUPPORTED" + entry["supports"].append({"polarity": None, "provenance": provenance, "unsupported": True}) + return entry["status"] + + def status(self, world: str, proposition: Term) -> str: + return self.entries.get(self._key(world, proposition), {}).get("status", "UNKNOWN") + + def supports(self, world: str, proposition: Term) -> list[dict[str, Any]]: + return list(self.entries.get(self._key(world, proposition), {}).get("supports", ())) + + def derive_across(self, source_world: str, target_world: str, operator: str, + proposition: Term, polarity: bool, provenance: str) -> str: + """The ONLY path by which content crosses a world boundary.""" + licensed = REGISTERED_CROSSINGS.get((source_world, operator)) + if licensed != target_world: + raise WorldBoundaryError( + f"no registered operator carries {source_world} -> {target_world} via {operator!r}") + if self.status(source_world, proposition) not in ("TRUE", "FALSE", "BOTH"): + raise WorldBoundaryError("cannot cross with an unasserted proposition") + return self.assert_(target_world, proposition, polarity, + f"{operator}({source_world}): {provenance}") + + +__all__ = [ + "App", "BASE_TYPES", "Base", "Const", "E", "EV", "EpistemicStore", "Fun", "GQ", + "Lam", "REGISTERED_CROSSINGS", "SCHEMA", "STATUSES", "T", "TM", "Term", "Type", + "TypeError_", "Var", "W", "WORLD_SCOPES", "WorldBoundaryError", "alpha_equal", + "canonical", "digest", "free_variables", "infer", "normalize", "pred", +] diff --git a/root/wander-train/stage5m13e6_realizer2.py b/root/wander-train/stage5m13e6_realizer2.py new file mode 100644 index 0000000000000000000000000000000000000000..2e539939df59edcdd836d31ecb24c436798b691c --- /dev/null +++ b/root/wander-train/stage5m13e6_realizer2.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Realizer2 — natural-register realization for K7 LinkedPrograms. + +Composes the sealed M13e4 Realizer for clause-level surfaces and adds the +linked/setting constructions. Round-trip law: realize2 output must re-parse +via K7 to a structurally identical program (checked by the external gate +battery in the K7 development evaluator, not assumed here). +""" + +from __future__ import annotations + +import sys + +sys.path.insert(0, r"C:\Users\BenHo\Desktop\ClaudeCode\_m13e0v4\control-install") + +from stage5m13e4_migration2 import Realizer, soften +from stage5m13e2_construction_kernel_k7 import LinkedProgram, CONNECTIVES + +RELATION_SURFACE = {value[0]: key for key, value in CONNECTIVES.items()} + + +def _decap(sentence: str) -> str: + words = sentence.split() + if words and words[0][0].isupper() and not words[0].isupper() and words[0] not in ("I",): + head = words[0] + # keep proper names capitalized: single capitalized token followed by + # lowercase verb is ambiguous; the sealed Realizer capitalizes clause + # starts, so decap only kernel-known lowercase-start surfaces. + if head.lower() in ("the", "a", "an", "every", "no", "some", "there", + "in", "on", "at", "after", "before", "during", "not"): + words[0] = head[0].lower() + head[1:] + return " ".join(words) + + +class Realizer2(Realizer): + def _cs(self, clause) -> str: + """Clause surface via the sealed realizer, with K7 adverbials re-applied.""" + surface = self.realize(clause).rstrip(".") + adverbial = (clause.predicate or {}).get("k7_adverbial") + if adverbial: + surface = f"{surface} {adverbial}" + pre_adverb = (clause.predicate or {}).get("k7_pre_adverb") + if pre_adverb: + words = surface.split() + # placement before the final verb word (intransitive clauses); + # registered limitation: transitive placement is approximate until + # Realizer2 exposes verb position. + words.insert(len(words) - 1, pre_adverb) + surface = " ".join(words) + return surface + + def realize_linked(self, program: LinkedProgram) -> str: + if program.relation == "apposition": + meta = program.meta + main_surface = self._cs(program.main) + name = meta.get("name", "") + if name and main_surface.lower().startswith(name.lower()): + rest = main_surface[len(name):].lstrip() + return f"{name}, {meta['article']} {meta['description']}, {rest}." + isa_surface = self._cs(program.subordinate) + return f"{main_surface}, and {_decap(isa_surface)}." + if program.relation == "setting": + setting = program.setting or {} + main_surface = self._cs(program.main) + return f"{setting.get('prep', 'in').capitalize()} {setting.get('np', '')}, {_decap(main_surface)}." + if program.relation == "located": + meta = program.meta + setting = program.setting or {} + entity_np = (meta.get("article", "") + " " + meta.get("entity_np", "")).strip() + copula = "was" if program.main.tense == "past" else "is" + if entity_np.split()[:1] and entity_np.split()[0] in ("three", "two", "four"): + copula = "were" if program.main.tense == "past" else "are" + surface = f"There {copula} {entity_np} {setting.get('prep', 'in')} {setting.get('np', '')}." + return surface[0].upper() + surface[1:] + if program.relation == "cleft_focus": + meta = program.meta + base = self._cs(program.main) + focus = meta["focus"] + if base.lower().startswith(focus.lower()): + rest = base[len(focus):].lstrip() + return f"It {meta['copula']} {focus} {meta['relativizer']} {rest}." + return f"{base}." + if program.relation == "attitude": + meta = program.meta + verb = {"say": "said", "believe": "believed", "think": "thought", + "claim": "claimed", "report": "reported"}[meta["verb_lemma"]] \ + if meta["tense"] == "past" else meta["verb_lemma"] + "s" + content = _decap(self._cs(program.main)) + return f"{meta['holder']} {verb} that {content}." + if program.relation == "neg_universal": + base = self._cs(program.main) + for head in ("Every ", "every "): + if base.startswith(head): + return f"Not every {base[len(head):]}." + return f"Not {base[0].lower()}{base[1:]}." + if program.relation == "possessive": + meta = program.meta + base = self._cs(program.main) + for head in (f"The {meta['head']}", f"the {meta['head']}"): + if base.startswith(head): + return f"{meta['chain']} {meta['head']}{base[len(head):]}." + return f"{base}." + if program.relation == "verbal_subject": + meta = program.meta + base = self._cs(program.main) + for head in ("The activity ", "the activity "): + if base.startswith(head): + return f"{meta['subject_np']} {base[len(head):]}." + return f"{base}." + if program.relation == "relative": + meta = program.meta + main_surface = self._cs(program.main) + head = meta["head"] + if main_surface.lower().startswith(head.lower()): + rest = main_surface[len(head):].lstrip() + return f"{head} {meta['relativizer']} {meta['rel_surface']} {rest}." + return f"{main_surface}." + if program.relation == "conjunction_ellipsis": + meta = program.meta + return f"{self._cs(program.main)} and {meta['right_surface']}." + if program.relation == "measure": + meta = program.meta + return (f"{meta['np']} {meta['copula']} {meta['value']} " + f"{meta['unit']} {meta['dimension']}.") + if program.relation == "phrasal": + meta = program.meta + return f"{meta['np']} {meta['surface_verb']}." + if program.relation == "passive_agent": + meta = program.meta + return (f"{meta['patient_np']} {meta['copula']} {meta['participle']} " + f"by {meta['agent_np']}.") + connective = RELATION_SURFACE.get(program.relation) + if connective is None: + return self.realize(program.main) + main_surface = self._cs(program.main) + sub_surface = self._cs(program.subordinate) + if program.surface_order == "fronted" and CONNECTIVES[connective][1]: + return f"{connective.capitalize()} {_decap(sub_surface)}, {_decap(main_surface)}." + joiner = ", so " if connective == "so" else f" {connective} " + return f"{main_surface}{joiner}{_decap(sub_surface)}." + + def realize2(self, program) -> str: + if isinstance(program, LinkedProgram): + surface = soften(self.realize_linked(program)) + return surface[0].upper() + surface[1:] if surface else surface + surface = self._cs(program).rstrip(".") + "." + return surface[0].upper() + surface[1:] diff --git a/root/wander-train/stage5m13e6_realizer2b.py b/root/wander-train/stage5m13e6_realizer2b.py new file mode 100644 index 0000000000000000000000000000000000000000..d8a33127cd3816a4f793104bd2bdf3723fd80538 --- /dev/null +++ b/root/wander-train/stage5m13e6_realizer2b.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Realizer2b — recursive realization for K7b composed programs. + +Additive successor to Realizer2 (frozen; not edited). _cs now dispatches: +a linked sub-program realizes through realize_linked recursively, a plain +clause through the sealed realizer. Adds the parenthetical attitude surface.""" + +from __future__ import annotations + +from stage5m13e2_construction_kernel_k7 import LinkedProgram +from stage5m13e6_realizer2 import Realizer2, _decap + + +class Realizer2b(Realizer2): + def _cs(self, clause) -> str: + if isinstance(clause, LinkedProgram): + return self.realize_linked(clause).rstrip(".") + surface = super()._cs(clause) + numeral = (clause.predicate or {}).get("k7_numeral_object") + if numeral: + surface = f"{surface} {numeral}" + return surface + + def realize_linked(self, program: LinkedProgram) -> str: + meta = program.meta or {} + if program.relation == "attitude" and meta.get("parenthetical"): + content = self._cs(program.main) + split_at = meta.get("split_at", "") + verb = {"say": "said", "believe": "believed", "claim": "claimed"}[meta["verb_lemma"]] \ + if meta["tense"] == "past" else meta["verb_lemma"] + "s" + if split_at and content.lower().startswith(split_at.lower()): + rest = content[len(split_at):].strip() + return f"{split_at}, {meta['holder']} {verb}, {rest}." + return f"{meta['holder']} {verb} that {_decap(content)}." + if program.relation == "located": + # tolerate a linked main (no .tense attribute) + tense = getattr(program.main, "tense", "present") + setting = program.setting or {} + entity_np = (meta.get("article", "") + " " + meta.get("entity_np", "")).strip() + copula = "was" if tense == "past" else "is" + if entity_np.split()[:1] and entity_np.split()[0] in ("three", "two", "four"): + copula = "were" if tense == "past" else "are" + surface = f"There {copula} {entity_np} {setting.get('prep', 'in')} {setting.get('np', '')}." + return surface[0].upper() + surface[1:] + return super().realize_linked(program) diff --git a/root/wander-train/stage5m13e8_paragraph_compiler_v1.py b/root/wander-train/stage5m13e8_paragraph_compiler_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..b34e82f96dd157d7199a55a7628db42150c96cee --- /dev/null +++ b/root/wander-train/stage5m13e8_paragraph_compiler_v1.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""H2 slice 1 — paragraph-program compiler (development evidence). + +Compiles a claim bundle (provenance-addressed sentences from one page/topic) +through a typed paragraph frame into a natural-register paragraph, and keeps +the program: every emitted sentence is a MOVE with its claim address, so +Program(paragraph) is recoverable from the output by construction. + +Moves implemented this slice: TOPIC (most definitional claim, attributed), +EVIDENCE (further claims, rotating natural attribution), BRIDGE (registered +connective to a named next topic). ANALYSIS moves are deliberately absent +until they can be engine-derived with premises — a templated analysis +sentence would violate the derived-provenance law. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field + +from stage5m13e5_frame_transform_v1 import classify_clause_initial_it + +NATURAL_FORMS = [ + "According to {page}, {claim}", + "The page for {page} records: {claim}", + "{page} notes that {claim}", + "As {page} has it, {claim}", + "From {page}: {claim}", +] + + +@dataclass +class ParagraphProgram: + topic: str + moves: list[dict] = field(default_factory=list) # {move, claim_id, page, text} + + def public(self) -> dict: + return {"topic": self.topic, "moves": self.moves} + + +def _definitional_score(topic: str, sentence: str) -> int: + low = sentence.lower() + t = topic.lower() + score = 0 + # a genuine definition: "(A|An|The)? TOPIC is a/an ..." near the start + if re.match(rf"^(?:a|an|the)?\s*{re.escape(t)}\b[^.]{{0,40}}?\bis (?:a|an)\b", low): + score += 6 + if low.startswith(t): + score += 2 + if re.search(r"\bis (?:a|an|the)\b", low): + score += 1 + if t in low: + score += 1 + return score + + +def compile_paragraph(topic: str, claims: list[dict], *, evidence: int = 3, + next_topic: str | None = None) -> tuple[str, ParagraphProgram]: + """claims: [{claim_id, page, sentence}] — provenance-addressed inputs only.""" + if not claims: + raise ValueError("empty claim bundle") + program = ParagraphProgram(topic=topic) + ranked = sorted(claims, key=lambda c: -_definitional_score(topic, c["sentence"])) + lead, rest = ranked[0], ranked[1:1 + evidence] + sentences = [] + # TOPIC move: attributed definitional lead (grounded referential `it` only — + # the P068 classifier guards substitution here exactly as in the transformer). + lead_text = lead["sentence"] + if classify_clause_initial_it(lead_text) == "referential": + lead_text = lead["page"] + lead_text[2:] + sentences.append(f"According to {lead['page']}, {lead_text} [{lead['claim_id']}]") + program.moves.append({"move": "TOPIC", "claim_id": lead["claim_id"], + "page": lead["page"], "text": lead_text}) + # EVIDENCE moves: rotating natural attribution + for index, claim in enumerate(rest): + form = NATURAL_FORMS[(index + 1) % len(NATURAL_FORMS)] + rendered = form.format(page=claim["page"], claim=claim["sentence"]) + sentences.append(f"{rendered} [{claim['claim_id']}]") + program.moves.append({"move": "EVIDENCE", "claim_id": claim["claim_id"], + "page": claim["page"], "text": claim["sentence"]}) + # BRIDGE move: registered connective, no factual content, so no address + if next_topic: + sentences.append(f"The record turns next to {next_topic}.") + program.moves.append({"move": "BRIDGE", "claim_id": None, + "page": None, "text": next_topic}) + return " ".join(sentences), program + + +def recover_program(paragraph: str, program: ParagraphProgram) -> bool: + """Round-trip law at paragraph scale: every addressed move's claim text and + address must be recoverable from the emitted surface, in move order.""" + cursor = 0 + for move in program.moves: + if move["claim_id"] is None: + continue + address = f"[{move['claim_id']}]" + position = paragraph.find(address, cursor) + if position < 0 or move["text"] not in paragraph[cursor:position]: + return False + cursor = position + return True diff --git a/root/wander-train/stage5m13e8_paragraph_compiler_v3.py b/root/wander-train/stage5m13e8_paragraph_compiler_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..12339b3317e5c1879a5d1cee7c0f19e6e39fb303 --- /dev/null +++ b/root/wander-train/stage5m13e8_paragraph_compiler_v3.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""H2 slice 2b — relevance-gated paragraph compiler (additive over v2). + +Claims pass the typed relevance gate BEFORE composition: only exact_topic and +explanatory claims may become TOPIC/EVIDENCE moves; every exclusion is +returned in a ledger with its class (nothing silently dropped, nothing +irrelevant attributed). With this gate in place, compiled paragraphs graduate +from 'development artifact' toward citable capability evidence per the +epistemic charter (final promotion still requires the H5 panel).""" + +from __future__ import annotations + +from stage5m13e8_paragraph_compiler_v2 import compile_paragraph_v2, recover_program_v2 +from stage5m13e8_relevance_gate_v1 import gate_bundle + + +def compile_gated_paragraph(topic: str, claims: list[dict], **kwargs): + admitted, excluded = gate_bundle(topic, claims) + if not admitted: + return None, None, excluded + paragraph, program = compile_paragraph_v2(topic, admitted, **kwargs) + return paragraph, program, excluded diff --git a/root/wander-train/stage5m17_release_arbiter_v1.py b/root/wander-train/stage5m17_release_arbiter_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..ee08fac4222e51b08a8a00a86f76c0ce4e256395 --- /dev/null +++ b/root/wander-train/stage5m17_release_arbiter_v1.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""M17a — release-runtime shadow arbiter (M-series operation families). + +Wraps the FROZEN K138 bid geometry (stage5k138_geometric_ingress_shadow_arbiter +.encode — role-bound word states + bigram bindings over unitary phasors; import +only, never edited) with the M-series release runtime's interpretation families. +Every family bids SIMULTANEOUSLY: bid = 1.4 * geometric resonance against the +family's prototype state + typed marker evidence. Winner/runner-up/margin with +the K138 clarification floor. Shadow mode: controls no dispatch. + +Prototype phrases are generic paraphrase templates authored for this module — +not panel surfaces, not residue surfaces (K138 law). No LM, no transformer, +no gradient; deterministic seeded phasors throughout. +""" + +from __future__ import annotations + +import hashlib +import re + +import numpy as np + +from stage5k138_geometric_ingress_shadow_arbiter import encode # frozen geometry +from stage5k138_typed_interpretation_candidate import ( + ArbitrationRecord, + InterpretationCandidate, +) + +CLARIFICATION_FLOOR = 0.55 +MARGIN_TAU = 0.08 + +M_FAMILIES = ( + "summarize", "compare", "list_n", "multi_step", "timeline", + "why_different", "what_if", "explain_simply", "definitional", + "act_refine", "act_continue", "act_provenance", "act_acknowledge", + "open_topic", "clarification", +) + + +def path_family(path: str | None) -> str | None: + """Map a runtime dispatch path to its interpretation family.""" + if not path: + return None + if path.startswith("act:refine"): + return "act_refine" + table = { + "act:continue_topic": "act_continue", + "act:provenance_replay": "act_provenance", + "act:acknowledge": "act_acknowledge", + "capability:summarize": "summarize", "anchored_summary": "summarize", + "capability:compare": "compare", + "capability:list_n": "list_n", + "capability:multi_step": "multi_step", + "capability:timeline": "timeline", + "capability:why_different": "why_different", + "capability:what_if": "what_if", + "capability:explain_simply": "explain_simply", + "definitional_encyclopedic": "definitional", + "conceptnet_commonsense": "open_topic", + "session_relational": "open_topic", + "document_runtime": "open_topic", + "gated_document": "open_topic", + } + return table.get(path) + + +PROTOTYPES: dict[str, tuple[str, ...]] = { + "summarize": ( + "summarize the topic for me", + "give me a brief summary of the subject", + "give me the short version of the topic", + "provide an overview of the subject", + "recap the main points about the topic", + ), + "compare": ( + "compare the first thing and the second thing", + "what is the difference between one thing and another", + "how do the two subjects differ from each other", + "contrast the first topic with the second topic", + ), + "list_n": ( + "list three facts about the topic", + "give me five facts about the subject", + "name a few things about the topic", + "tell me several points about the subject", + ), + "multi_step": ( + "summarize the first topic and then list facts about the second", + "define the term then compare two subjects", + "first give an overview then name some facts", + ), + "timeline": ( + "give me a timeline of the topic", + "show the chronology of the subject", + "walk me through the history year by year", + "lay out the dated events of the topic in order", + ), + "why_different": ( + "why are the two things different", + "explain why one subject differs from the other", + "how come the first thing and the second thing are so different", + ), + "what_if": ( + "what if the thing had no part", + "what would happen if the subject lost its feature", + "suppose the thing had no part what then", + "imagine the subject without its feature", + ), + "explain_simply": ( + "explain the topic simply", + "explain the subject in simple terms", + "break the topic down in plain language", + "explain the idea so a child could follow it", + ), + "definitional": ( + "what is a concept", + "what does the term mean", + "define the following word", + "what are those things exactly", + ), + "act_refine": ( + "simpler", + "make it simpler", + "shorter please", + "say that more simply", + "again but simpler", + ), + "act_continue": ( + "tell me more", + "keep going", + "go on", + "tell me more about that", + "continue please", + ), + "act_provenance": ( + "why did you say that", + "how do you know that", + "where did that answer come from", + "what is the source for that", + ), + "act_acknowledge": ( + "thanks that helped", + "thank you", + "great answer cheers", + "nice that was helpful", + ), + "open_topic": ( + "tell me about a broad topic", + "share background knowledge on the subject", + "talk to me about the matter", + "what do you know about the topic", + ), +} + +NUMBER_WORDS = {"one", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "few", "couple", "several", "some"} +LIST_NOUNS = {"facts", "things", "points", "items", "examples", "reasons"} +SUMMARY_TOKENS = {"summarize", "summarise", "summary", "overview", "recap", "rundown"} +COMPARE_TOKENS = {"compare", "contrast", "versus", "vs", "difference", "differences", "differ"} +SIMPLE_TOKENS = {"simply", "simple", "simpler", "plain", "plainly", "easy", "easier"} +REFINE_TOKENS = {"simpler", "shorter", "briefer"} +ACK_TOKENS = {"thanks", "thank", "cheers", "helpful", "perfect", "great", "nice"} +TIMELINE_TOKENS = {"timeline", "chronology", "chronological"} +DIFFERENT_TOKENS = {"different", "differs", "differ", "differently"} + +PROVENANCE_PHRASES = ("why did you say", "how do you know", "where did that", + "what is the source", "what's the source", "came from") +CONTINUE_PHRASES = ("tell me more", "keep going", "go on", "continue", "elaborate") +WHATIF_PHRASES = ("what if", "what would happen if", "suppose ", "imagine ") +DEFINITION_LEADS = ("what is ", "what are ", "define ", "what does ") +TELL_ABOUT_LEADS = ("tell me about ", "talk to me about ", "what do you know about ") + + +def _terms(text: str) -> list[str]: + return re.findall(r"[a-z0-9][a-z0-9'-]*", text.casefold()) + + +def features(text: str) -> dict[str, float]: + lower = " ".join(text.casefold().strip().split()) + tokens = _terms(lower) + token_set = set(tokens) + first = tokens[0] if tokens else "" + why_lead = first == "why" or lower.startswith("how come") + count_present = bool(token_set & NUMBER_WORDS) or any(t.isdigit() for t in tokens) + sentence_break_then = bool(re.search(r"[,.?!;] *(and )?then ", lower)) or " and then " in lower + refine_phrase = "more simply" in lower or "more briefly" in lower + return { + "question": float(lower.endswith("?")), + "summary_token": float(bool(token_set & SUMMARY_TOKENS) or "short version" in lower + or "brief version" in lower or "sum up" in lower), + "compare_token": float(bool(token_set & COMPARE_TOKENS)), + "list_count": float(count_present and bool(token_set & LIST_NOUNS)), + "timeline_token": float(bool(token_set & TIMELINE_TOKENS) or "year by year" in lower + or "in order" in lower and "events" in lower), + "why_lead": float(why_lead), + "different_token": float(bool(token_set & DIFFERENT_TOKENS)), + "whatif_marker": float(any(lower.startswith(p) or f" {p}" in lower + for p in WHATIF_PHRASES)), + "simple_token": float(bool(token_set & SIMPLE_TOKENS) or "in simple terms" in lower + or "five year old" in lower or "a child" in lower + or "new to" in lower or "for beginners" in lower), + "refine_short": float(len(tokens) <= 5 + and (bool(token_set & REFINE_TOKENS) or refine_phrase)), + "ack_marker": float(len(tokens) <= 6 and bool(token_set & ACK_TOKENS)), + "provenance_phrase": float(any(p in lower for p in PROVENANCE_PHRASES)), + "continue_phrase": float(any(p in lower for p in CONTINUE_PHRASES) + and "about" not in lower.split("more")[0]), + "chain_marker": float(sentence_break_then), + "definition_lead": float(lower.startswith(DEFINITION_LEADS)), + "tell_about_lead": float(lower.startswith(TELL_ABOUT_LEADS) + and not any(p in lower for p in CONTINUE_PHRASES)), + "very_short": float(len(tokens) <= 3), + } + + +FEATURE_WEIGHTS: dict[str, dict[str, float]] = { + "summarize": {"summary_token": 1.00, "list_count": -0.40, "chain_marker": -0.70, + "timeline_token": -0.50, "refine_short": -0.60}, + "compare": {"compare_token": 0.90, "why_lead": -0.70, "chain_marker": -0.70, + "whatif_marker": -0.60}, + "list_n": {"list_count": 1.00, "chain_marker": -0.70, "summary_token": -0.30}, + "multi_step": {"chain_marker": 1.20}, + "timeline": {"timeline_token": 1.00, "chain_marker": -0.70}, + "why_different": {"why_lead": 0.55, "different_token": 0.55, "compare_token": 0.10, + "whatif_marker": -0.60, "chain_marker": -0.70}, + "what_if": {"whatif_marker": 1.00, "chain_marker": -0.70}, + "explain_simply": {"simple_token": 0.95, "refine_short": -0.50, "chain_marker": -0.70}, + "definitional": {"definition_lead": 0.85, "compare_token": -0.80, "chain_marker": -0.70, + "whatif_marker": -0.80, "different_token": -0.50, "simple_token": -0.40, + "list_count": -0.60, "timeline_token": -0.60, "summary_token": -0.50}, + "act_refine": {"refine_short": 1.10, "simple_token": 0.15, "question": -0.30}, + "act_continue": {"continue_phrase": 1.00, "very_short": 0.15}, + "act_provenance": {"provenance_phrase": 1.10}, + "act_acknowledge": {"ack_marker": 1.00, "question": -0.50}, + "open_topic": {"tell_about_lead": 0.80, "definition_lead": -0.40, "summary_token": -0.40, + "compare_token": -0.40, "whatif_marker": -0.50, "simple_token": -0.40, + "timeline_token": -0.50, "list_count": -0.50, "chain_marker": -0.50, + "refine_short": -0.60, "ack_marker": -0.60, "provenance_phrase": -0.60}, +} + +_PROTOTYPE_STATES: dict[str, np.ndarray] = {} + + +def prototype(family: str) -> np.ndarray: + state = _PROTOTYPE_STATES.get(family) + if state is None: + phrases = PROTOTYPES.get(family, ()) + stacked = np.sum([encode(phrase) for phrase in phrases], axis=0) + norm = np.linalg.norm(stacked) + state = stacked / norm if norm else stacked + _PROTOTYPE_STATES[family] = state + return state + + +def resonance(bundle: np.ndarray, family: str) -> float: + state = prototype(family) + if not np.any(state): + return 0.0 + return float(np.real(np.vdot(state, bundle))) + + +def arbitrate(text: str) -> ArbitrationRecord: + bundle = encode(text) + evidence = features(text) + candidates: list[InterpretationCandidate] = [] + for family in M_FAMILIES: + if family == "clarification": + continue + weights = FEATURE_WEIGHTS.get(family, {}) + typed = {name: weights[name] * evidence[name] + for name in weights if evidence.get(name)} + geometric = resonance(bundle, family) + bid = 1.4 * geometric + sum(typed.values()) + candidates.append(InterpretationCandidate(family, geometric, typed, bid)) + candidates.sort(key=lambda value: (-value.bid, value.family)) + top, second = candidates[0], candidates[1] + margin = top.bid - second.bid + floor_applied = top.bid < CLARIFICATION_FLOOR or margin < MARGIN_TAU + winner = "clarification" if floor_applied else top.family + runner_up = top.family if floor_applied else second.family + return ArbitrationRecord( + utterance_sha256=hashlib.sha256(text.encode()).hexdigest(), + candidates=tuple(candidates), + winner=winner, + runner_up=runner_up, + margin=margin, + clarification_floor_applied=floor_applied, + ) + + +__all__ = ["CLARIFICATION_FLOOR", "MARGIN_TAU", "M_FAMILIES", + "arbitrate", "features", "path_family", "prototype", "resonance"] diff --git a/root/wander-train/stage5m17_release_arbiter_v2.py b/root/wander-train/stage5m17_release_arbiter_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..9f9f33c6390b94cda42ac2d045ce905289da48a6 --- /dev/null +++ b/root/wander-train/stage5m17_release_arbiter_v2.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""M17b2 — release arbiter v2 (additive successor to v1; v1 untouched). + +Two repairs from the sealed M17b one-shot failure (see +stage5m17b_oneshot_adjudication_v1.md): + +1. WIDENED PROTOTYPES — lexically diverse generic paraphrase templates per + family (gist/nutshell/story/not-the-same/appreciated classes), so the + frozen K138 geometry has token support across more of the paraphrase space. +2. CONCEPTNET MARKER PROJECTION — novel content tokens are mapped into family + marker classes through the admitted ConceptNet English edges (Synonym / + FormOf / SimilarTo / IsA / DerivedFrom, weight-ordered, 1 hop, read-only). + Corpus-grounded: "fistful" reaches the count class because the admitted + corpus says Synonym(fistful, handful), not because an author listed it. + +Same bid law as v1: 1.4 * resonance + typed markers; same floor and tau. +No LM, no transformer, no gradient.""" + +from __future__ import annotations + +import hashlib +import re +import sqlite3 +from pathlib import Path + +import numpy as np + +from stage5k138_geometric_ingress_shadow_arbiter import encode # frozen geometry +from stage5k138_typed_interpretation_candidate import ( + ArbitrationRecord, + InterpretationCandidate, +) +from stage5m17_release_arbiter_v1 import ( + CLARIFICATION_FLOOR, + MARGIN_TAU, + M_FAMILIES, + FEATURE_WEIGHTS, + PROTOTYPES as PROTOTYPES_V1, + features as features_v1, + path_family, +) + +CONCEPTNET_DB = (Path(__file__).resolve().parents[1] / "fable-content" + / "oracle-m-series-2026-08-15" / "m13e5_conceptnet_build_a_v1" + / "conceptnet_specialist.sqlite3") +PROJECTION_RELS = ("Synonym", "FormOf", "SimilarTo", "IsA", "DerivedFrom") + +EXTRA_PROTOTYPES: dict[str, tuple[str, ...]] = { + "summarize": ( + "give me the gist of the topic", + "the topic in a nutshell please", + "a brief overview of the subject please", + "boil the topic down to its essentials", + "what is the story of the subject in brief", + ), + "compare": ( + "how are the first thing and the second thing not the same", + "what do the two subjects have in common and where do they part", + "how does one thing stack up against the other", + ), + "list_n": ( + "name a handful of facts about the topic", + "give me a smattering of points about the subject", + "rattle off some things about the topic", + ), + "timeline": ( + "what is the story of the topic over the years", + "how did the subject unfold through time", + "trace the subject from its beginning to now", + ), + "explain_simply": ( + "explain the topic in plain words", + "put the subject in terms a beginner would get", + "make the idea easy to understand", + ), + "definitional": ( + "what would you say a concept is", + "what does the word actually mean", + "how would you define the term", + ), + "act_provenance": ( + "why would you claim that", + "what makes you say that", + "on what basis did you answer", + ), + "act_acknowledge": ( + "much appreciated", + "cheers i am grateful", + "that was really helpful thanks a lot", + ), +} + +# family marker classes: seed words; ConceptNet projects novel tokens into them +MARKER_CLASSES: dict[str, set[str]] = { + "summary": {"summarize", "summarise", "summary", "overview", "recap", + "rundown", "gist", "synopsis", "digest", "outline", "essentials"}, + "compare": {"compare", "contrast", "difference", "differences", "differ", + "versus", "alike", "dissimilar"}, + "count": {"few", "couple", "several", "handful", "some", "quantity", + "number", "fistful", "smattering"}, + "list": {"facts", "things", "points", "items", "examples", "reasons"}, + "timeline": {"timeline", "chronology", "chronological", "history"}, + "simple": {"simple", "simply", "plain", "easy", "easier", "basic", + "beginner", "clear"}, + "ack": {"thanks", "thank", "appreciate", "appreciated", "grateful", + "gratitude", "cheers", "helpful"}, + "provenance": {"claim", "basis", "source", "evidence", "say", "said", + "assert", "assertion", "know"}, + "definitional": {"mean", "meaning", "define", "definition"}, +} + +STOPWORDS = {"a", "an", "the", "of", "to", "for", "in", "on", "and", "or", + "is", "are", "was", "be", "this", "that", "these", "those", "it", + "its", "you", "your", "me", "my", "i", "we", "what", "whats", + "how", "why", "would", "could", "should", "do", "does", "did", + "give", "get", "please", "about", "with", "from", "so", "not"} + +_connection: sqlite3.Connection | None = None +_projection_cache: dict[str, frozenset[str]] = {} + + +def _db() -> sqlite3.Connection | None: + global _connection + if _connection is None and CONCEPTNET_DB.exists(): + _connection = sqlite3.connect(f"file:{CONCEPTNET_DB}?mode=ro", uri=True) + return _connection + + +def project_token(token: str) -> frozenset[str]: + """Marker classes reachable from token via 1 admitted ConceptNet hop.""" + cached = _projection_cache.get(token) + if cached is not None: + return cached + classes: set[str] = set() + connection = _db() + if connection is not None: + placeholders = ",".join("?" for _ in PROJECTION_RELS) + rows = connection.execute( + f"SELECT start, end FROM edge WHERE (start=? OR end=?) " + f"AND rel IN ({placeholders}) ORDER BY weight DESC LIMIT 24", + (token, token, *PROJECTION_RELS)).fetchall() + neighbors: set[str] = set() + for start, end in rows: + other = end if start == token else start + neighbors.update(other.split()) # multiword ends contribute words + for name, seeds in MARKER_CLASSES.items(): + if neighbors & seeds: + classes.add(name) + result = frozenset(classes) + _projection_cache[token] = result + return result + + +def _terms(text: str) -> list[str]: + return [w.strip("'-") for w in re.findall(r"[a-z0-9][a-z0-9'-]*", text.casefold())] + + +def features(text: str) -> dict[str, float]: + base = features_v1(text) + lower = " ".join(text.casefold().strip().split()) + tokens = _terms(lower) + token_set = set(tokens) + first = tokens[0] if tokens else "" + + hits: set[str] = set() + for name, seeds in MARKER_CLASSES.items(): + if token_set & seeds: + hits.add(name) + for token in token_set: + if len(token) >= 3 and token not in STOPWORDS: + hits.update(project_token(token)) + + if "summary" in hits or "in a nutshell" in lower or "boil" in lower: + base["summary_token"] = 1.0 + if ("compare" in hits or "not the same" in lower or "not alike" in lower + or "in common" in lower): + base["compare_token"] = 1.0 + if "count" in hits and "list" in hits: + base["list_count"] = 1.0 + if ("timeline" in hits or "over the years" in lower + or "through the years" in lower or "through time" in lower): + base["timeline_token"] = 1.0 + if "simple" in hits or "plain words" in lower: + base["simple_token"] = 1.0 + if len(tokens) <= 6 and "ack" in hits and not lower.endswith("?"): + base["ack_marker"] = 1.0 + if (first in ("why", "how") and "you" in token_set and "provenance" in hits + and len(tokens) <= 7): + base["provenance_phrase"] = 1.0 + if lower.endswith("?") and tokens and tokens[-1] == "is": + base["definition_lead"] = 1.0 # "what would you say a glacier is?" + if "definitional" in hits and lower.endswith("?") and "what" in token_set: + base["definition_lead"] = 1.0 + return base + + +PROTOTYPES: dict[str, tuple[str, ...]] = { + family: tuple(PROTOTYPES_V1.get(family, ())) + tuple(EXTRA_PROTOTYPES.get(family, ())) + for family in set(PROTOTYPES_V1) | set(EXTRA_PROTOTYPES) +} + +_prototype_states: dict[str, np.ndarray] = {} + + +def prototype(family: str) -> np.ndarray: + state = _prototype_states.get(family) + if state is None: + phrases = PROTOTYPES.get(family, ()) + stacked = np.sum([encode(phrase) for phrase in phrases], axis=0) + norm = np.linalg.norm(stacked) + state = stacked / norm if norm else stacked + _prototype_states[family] = state + return state + + +def resonance(bundle: np.ndarray, family: str) -> float: + state = prototype(family) + if not np.any(state): + return 0.0 + return float(np.real(np.vdot(state, bundle))) + + +def arbitrate(text: str) -> ArbitrationRecord: + bundle = encode(text) + evidence = features(text) + candidates: list[InterpretationCandidate] = [] + for family in M_FAMILIES: + if family == "clarification": + continue + weights = FEATURE_WEIGHTS.get(family, {}) + typed = {name: weights[name] * evidence[name] + for name in weights if evidence.get(name)} + geometric = resonance(bundle, family) + bid = 1.4 * geometric + sum(typed.values()) + candidates.append(InterpretationCandidate(family, geometric, typed, bid)) + candidates.sort(key=lambda value: (-value.bid, value.family)) + top, second = candidates[0], candidates[1] + margin = top.bid - second.bid + floor_applied = top.bid < CLARIFICATION_FLOOR or margin < MARGIN_TAU + winner = "clarification" if floor_applied else top.family + runner_up = top.family if floor_applied else second.family + return ArbitrationRecord( + utterance_sha256=hashlib.sha256(text.encode()).hexdigest(), + candidates=tuple(candidates), + winner=winner, + runner_up=runner_up, + margin=margin, + clarification_floor_applied=floor_applied, + ) + + +__all__ = ["arbitrate", "features", "path_family", "project_token", + "prototype", "resonance", "M_FAMILIES"] diff --git a/root/wander-train/stage5m17_release_arbiter_v3.py b/root/wander-train/stage5m17_release_arbiter_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..5e75df4dd29e188d7df7fcf9c9f8dbd3b56324f8 --- /dev/null +++ b/root/wander-train/stage5m17_release_arbiter_v3.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""M17b3 — release arbiter v3 (additive successor to v2; v1/v2 untouched). + +Four repairs from the sealed M17b2 one-shot (see +stage5m17b2_oneshot_adjudication_v1.md): + +1. SEED-EXCLUSIVITY — a token that is itself a seed of one marker class never + projects into another class ("define" is a definitional seed; its + ConceptNet neighborhood may not vote for summarize). The K161 + wrong-domain-monopoly bound, applied to projection. +2. DEFINITIONAL CUE — "define"/"definition" token + question fires the + definitional lead independent of "what". +3. TIMELINE SEEDS widened: beginning/beginnings/origins/start/unfold. +4. PROJECTION HYGIENE — hyper-common polysemous tokens are never projected + (like/now/get/...), and projection edges require weight >= 2.0. + +Bid law, floor, tau unchanged. No LM, no transformer, no gradient.""" + +from __future__ import annotations + +import hashlib +import re +import sqlite3 +from pathlib import Path + +import numpy as np + +from stage5k138_geometric_ingress_shadow_arbiter import encode # frozen geometry +from stage5k138_typed_interpretation_candidate import ( + ArbitrationRecord, + InterpretationCandidate, +) +from stage5m17_release_arbiter_v1 import ( + CLARIFICATION_FLOOR, + MARGIN_TAU, + M_FAMILIES, + FEATURE_WEIGHTS, + features as features_v1, + path_family, +) +from stage5m17_release_arbiter_v2 import ( + CONCEPTNET_DB, + PROJECTION_RELS, + PROTOTYPES, + MARKER_CLASSES as MARKER_CLASSES_V2, + STOPWORDS, +) + +MARKER_CLASSES: dict[str, set[str]] = {name: set(seeds) + for name, seeds in MARKER_CLASSES_V2.items()} +MARKER_CLASSES["timeline"] |= {"beginning", "beginnings", "origins", "start", + "started", "unfold", "unfolded"} +ALL_SEEDS: set[str] = set().union(*MARKER_CLASSES.values()) + +PROJECTION_STOP = { + "like", "now", "get", "got", "put", "keep", "kept", "make", "made", + "take", "took", "way", "ways", "kind", "kinds", "thing", "things", + "going", "come", "came", "look", "want", "need", "see", "sure", "well", + "right", "life", "back", "off", "over", "trace", "just", "really", +} + +_connection: sqlite3.Connection | None = None +_projection_cache: dict[str, frozenset[str]] = {} + + +def _db() -> sqlite3.Connection | None: + global _connection + if _connection is None and CONCEPTNET_DB.exists(): + _connection = sqlite3.connect(f"file:{CONCEPTNET_DB}?mode=ro", uri=True) + return _connection + + +def project_token(token: str) -> frozenset[str]: + """Marker classes reachable via 1 admitted ConceptNet hop (weight>=2.0). + Seed tokens and hyper-common tokens never project (rules 1 and 4).""" + if token in ALL_SEEDS or token in PROJECTION_STOP: + return frozenset() + cached = _projection_cache.get(token) + if cached is not None: + return cached + classes: set[str] = set() + connection = _db() + if connection is not None: + placeholders = ",".join("?" for _ in PROJECTION_RELS) + rows = connection.execute( + f"SELECT start, end FROM edge WHERE (start=? OR end=?) " + f"AND rel IN ({placeholders}) AND weight >= 2.0 " + f"ORDER BY weight DESC LIMIT 24", + (token, token, *PROJECTION_RELS)).fetchall() + neighbors: set[str] = set() + for start, end in rows: + other = end if start == token else start + neighbors.update(other.split()) + for name, seeds in MARKER_CLASSES.items(): + if neighbors & seeds: + classes.add(name) + result = frozenset(classes) + _projection_cache[token] = result + return result + + +def _terms(text: str) -> list[str]: + return [w.strip("'-") for w in re.findall(r"[a-z0-9][a-z0-9'-]*", text.casefold())] + + +def features(text: str) -> dict[str, float]: + base = features_v1(text) + lower = " ".join(text.casefold().strip().split()) + tokens = _terms(lower) + token_set = set(tokens) + first = tokens[0] if tokens else "" + + hits: set[str] = set() + for name, seeds in MARKER_CLASSES.items(): + if token_set & seeds: + hits.add(name) + for token in token_set: + if len(token) >= 3 and token not in STOPWORDS: + hits.update(project_token(token)) + + if "summary" in hits or "in a nutshell" in lower or "boil" in lower: + base["summary_token"] = 1.0 + if ("compare" in hits or "not the same" in lower or "not alike" in lower + or "in common" in lower): + base["compare_token"] = 1.0 + if "count" in hits and "list" in hits: + base["list_count"] = 1.0 + if ("timeline" in hits or "over the years" in lower + or "through the years" in lower or "through time" in lower): + base["timeline_token"] = 1.0 + if "simple" in hits or "plain words" in lower: + base["simple_token"] = 1.0 + if len(tokens) <= 6 and "ack" in hits and not lower.endswith("?"): + base["ack_marker"] = 1.0 + if (first in ("why", "how") and "you" in token_set and "provenance" in hits + and len(tokens) <= 7): + base["provenance_phrase"] = 1.0 + if lower.endswith("?") and tokens and tokens[-1] == "is": + base["definition_lead"] = 1.0 + if "definitional" in hits and lower.endswith("?") and "what" in token_set: + base["definition_lead"] = 1.0 + # repair 2: define/definition + question is definitional regardless of "what" + if ({"define", "definition", "defined"} & token_set) and lower.endswith("?"): + base["definition_lead"] = 1.0 + base["summary_token"] = 0.0 + return base + + +_prototype_states: dict[str, np.ndarray] = {} + + +def prototype(family: str) -> np.ndarray: + state = _prototype_states.get(family) + if state is None: + phrases = PROTOTYPES.get(family, ()) + stacked = np.sum([encode(phrase) for phrase in phrases], axis=0) + norm = np.linalg.norm(stacked) + state = stacked / norm if norm else stacked + _prototype_states[family] = state + return state + + +def resonance(bundle: np.ndarray, family: str) -> float: + state = prototype(family) + if not np.any(state): + return 0.0 + return float(np.real(np.vdot(state, bundle))) + + +def arbitrate(text: str) -> ArbitrationRecord: + bundle = encode(text) + evidence = features(text) + candidates: list[InterpretationCandidate] = [] + for family in M_FAMILIES: + if family == "clarification": + continue + weights = FEATURE_WEIGHTS.get(family, {}) + typed = {name: weights[name] * evidence[name] + for name in weights if evidence.get(name)} + geometric = resonance(bundle, family) + bid = 1.4 * geometric + sum(typed.values()) + candidates.append(InterpretationCandidate(family, geometric, typed, bid)) + candidates.sort(key=lambda value: (-value.bid, value.family)) + top, second = candidates[0], candidates[1] + margin = top.bid - second.bid + floor_applied = top.bid < CLARIFICATION_FLOOR or margin < MARGIN_TAU + winner = "clarification" if floor_applied else top.family + runner_up = top.family if floor_applied else second.family + return ArbitrationRecord( + utterance_sha256=hashlib.sha256(text.encode()).hexdigest(), + candidates=tuple(candidates), + winner=winner, + runner_up=runner_up, + margin=margin, + clarification_floor_applied=floor_applied, + ) + + +__all__ = ["arbitrate", "features", "path_family", "project_token", + "prototype", "resonance", "M_FAMILIES"] diff --git a/root/wander-train/stage5m21_transitive_reasoner_v1.py b/root/wander-train/stage5m21_transitive_reasoner_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5bca1d4df457179b9fadb2c07eb3dd6be48e82 --- /dev/null +++ b/root/wander-train/stage5m21_transitive_reasoner_v1.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""M21 — implied-knowledge engine, slice 1: transitive kind (IsA) chaining. + +The Oracle answers only when a single admitted claim matches, which is why +open-domain coverage is ~1%. But admitted claims IMPLY others: if the record +holds "a robin is a bird" and "a bird is an animal", it implicitly holds "a +robin is an animal" — it just never computes it. This module computes exactly +that closure, over the transitive IsA relation of the admitted ConceptNet +corpus, and returns the FULL premise chain so a derived answer stays auditable: +the answer is the shown chain, not a bare assertion. + +Target-anchored: given (X, Y) it searches for a shortest chain X -> ... -> Y, +breadth-first (shortest path first), each hop a real admitted edge. It never +enumerates a closure and asserts from it; it only confirms the specific link +the user asked about, and shows the steps. Bounded: depth<=MAX_DEPTH, a frontier +cap, a per-node edge cap; deterministic (edges are weight-then-lexical ordered). + +Known limitation, recorded for the next slice: chains can cross a word-sense +boundary (robin->thrush[bird]->candidiasis[disease]) because ConceptNet +conflates senses. This slice shows the chain rather than hiding it; sense- +consistent chaining is the successor (it reuses the M17c evidence-conditioned +sense machinery). No neural network, no gradient; pure graph search.""" + +from __future__ import annotations + +from collections import deque +from typing import Any + +MAX_DEPTH = 3 +FRONTIER_CAP = 400 +EDGE_CAP = 8 +MIN_WEIGHT = 1.0 +# A transitive derivation "X is a Y" only fires when Y is an ESTABLISHED +# category — many admitted claims call something a Y. This is the reasoning- +# level guard against sense-crossing chains (robin->thrush[bird]->candidiasis +# [disease]): the disease terminus has almost no incoming IsA edges, a real +# category has hundreds. Direct single edges are unaffected (they are citations, +# not derivations). Calibrated against the measured gap: traps <=13, the +# lowest kept real category 68. +CATEGORY_MIN_INDEGREE = 30 + + +def _norm(concept: str) -> str: + c = concept.strip().lower() + # a bare trailing plural is the common surface form ("are robins animals") + if c.endswith("s") and not c.endswith("ss") and len(c) > 3: + return c[:-1] + return c + + +def direct_isa(adapter: Any, x: str, y: str) -> dict | None: + """A single admitted IsA(x, y) edge, if present — a citation, not a chain.""" + xn, yn = _norm(x), _norm(y) + for concept in (x.strip().lower(), xn): + for edge in adapter.edges(concept, "IsA", limit=24): + end = edge["end"].strip().lower() + if end == y.strip().lower() or _norm(end) == yn: + return edge + return None + + +def category_indegree(adapter: Any, y: str) -> int: + """How many admitted IsA edges point AT y — how established a category it is. + Read-only against the same corpus; max over the surface and normalized form.""" + best = 0 + for form in dict.fromkeys([y.strip().lower(), _norm(y)]): + row = adapter.connection.execute( + "SELECT COUNT(*) FROM edge WHERE end=? AND rel='IsA'", (form,)).fetchone() + best = max(best, int(row[0]) if row else 0) + return best + + +def chain_isa(adapter: Any, x: str, y: str, *, max_depth: int = MAX_DEPTH) -> list[dict] | None: + """Shortest chain of admitted IsA edges from x to y, or None. + + BFS so the returned path is the fewest hops; the per-node edges are already + weight-descending, so among equal-length paths the strongest dominant-sense + links are explored first. Never returns a zero-hop (direct) path — that is + a citation, handled by direct_isa(). A chain is only returned when y is an + established category (guard against sense-crossing termini).""" + if category_indegree(adapter, y) < CATEGORY_MIN_INDEGREE: + return None + targets = {y.strip().lower(), _norm(y)} + starts = [x.strip().lower(), _norm(x)] + visited: set[str] = set(starts) + queue: deque[tuple[str, list[dict]]] = deque((s, []) for s in dict.fromkeys(starts)) + expanded = 0 + while queue and expanded < FRONTIER_CAP: + node, path = queue.popleft() + if len(path) >= max_depth: + continue + expanded += 1 + for edge in adapter.edges(node, "IsA", limit=EDGE_CAP): + if edge["weight"] < MIN_WEIGHT: + continue + end = edge["end"].strip().lower() + new_path = path + [edge] + if end in targets or _norm(end) in targets: + if len(new_path) >= 2: # >=2 hops = a genuine derivation + return new_path + # 1-hop match here would be a direct edge; let direct_isa own it + continue + if end not in visited: + visited.add(end) + queue.append((end, new_path)) + return None + + +def render_chain(adapter: Any, path: list[dict]) -> str: + """Human-readable step list: 'a robin is a bird; a bird is an animal'.""" + steps = [] + for edge in path: + rendered = adapter.render_edge(edge).rstrip(".") + steps.append(rendered[0].lower() + rendered[1:]) + return "; ".join(steps) diff --git a/root/wander-train/stage5m21_transitive_reasoner_v2.py b/root/wander-train/stage5m21_transitive_reasoner_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..ecb96515546779f1cb6980492c49a1d0299820b5 --- /dev/null +++ b/root/wander-train/stage5m21_transitive_reasoner_v2.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""M21 slice 1b — robust transitive-kind reasoner (additive successor to v1). + +Two robustness upgrades over v1, both root-caused from the v1 one-shot's +false abstentions and preserved trap: + +1. SENSE-DOMINANCE PRUNING replaces the blunt in-degree gate as the primary + guard. A sense-crossing chain (robin[bird] -> thrush -> candidiasis[disease]) + always enters through a MINORITY-sense edge: robin's dominant kind is "bird" + (weight 6.3); "thrush" is a weak 1.0 edge. So we only follow IsA edges whose + weight is >= max(1.0, DOMINANCE_FRAC x the node's strongest IsA weight). This + targets the actual trap mechanism instead of guessing at the terminus, and it + RECOVERS true chains the in-degree gate wrongly blocked (sparrow -> bird -> + vertebrate: vertebrate's in-degree 27 fell just under v1's threshold 30). + +2. A low category FLOOR (in-degree >= 12) remains as belt-and-suspenders so a + derivation never runs toward a tiny leaf concept, but it no longer does the + sense work — the dominance prune does. + +Same target-anchored BFS (shortest chain first), depth<=3, provenance chain +returned so the answer stays the shown derivation. No neural network.""" + +from __future__ import annotations + +from collections import deque +from typing import Any + +MAX_DEPTH = 3 +FRONTIER_CAP = 400 +EDGE_CAP = 10 +MIN_WEIGHT = 1.0 +DOMINANCE_FRAC = 0.25 # prune edges weaker than this x the node's top IsA edge +CATEGORY_MIN_INDEGREE = 12 # terminus must be a real (if small) category + + +def _norm(concept: str) -> str: + c = concept.strip().lower() + if c.endswith("s") and not c.endswith("ss") and len(c) > 3: + return c[:-1] + return c + + +def category_indegree(adapter: Any, y: str) -> int: + best = 0 + for form in dict.fromkeys([y.strip().lower(), _norm(y)]): + row = adapter.connection.execute( + "SELECT COUNT(*) FROM edge WHERE end=? AND rel='IsA'", (form,)).fetchone() + best = max(best, int(row[0]) if row else 0) + return best + + +def direct_isa(adapter: Any, x: str, y: str) -> dict | None: + xn, yn = _norm(x), _norm(y) + for concept in dict.fromkeys([x.strip().lower(), xn]): + for edge in adapter.edges(concept, "IsA", limit=24): + end = edge["end"].strip().lower() + if end == y.strip().lower() or _norm(end) == yn: + return edge + return None + + +def _dominant_edges(adapter: Any, node: str) -> list[dict]: + """IsA edges of node, minority-sense edges pruned by relative weight.""" + edges = adapter.edges(node, "IsA", limit=EDGE_CAP) + if not edges: + return [] + top = max(e["weight"] for e in edges) + threshold = max(MIN_WEIGHT, DOMINANCE_FRAC * top) + return [e for e in edges if e["weight"] >= threshold] + + +def chain_isa(adapter: Any, x: str, y: str, *, max_depth: int = MAX_DEPTH) -> list[dict] | None: + """Shortest dominant-sense chain of admitted IsA edges from x to y, or None.""" + if category_indegree(adapter, y) < CATEGORY_MIN_INDEGREE: + return None + targets = {y.strip().lower(), _norm(y)} + starts = [x.strip().lower(), _norm(x)] + visited: set[str] = set(starts) + queue: deque[tuple[str, list[dict]]] = deque((s, []) for s in dict.fromkeys(starts)) + expanded = 0 + while queue and expanded < FRONTIER_CAP: + node, path = queue.popleft() + if len(path) >= max_depth: + continue + expanded += 1 + for edge in _dominant_edges(adapter, node): + end = edge["end"].strip().lower() + new_path = path + [edge] + if (end in targets or _norm(end) in targets) and len(new_path) >= 2: + return new_path + if end not in visited: + visited.add(end) + queue.append((end, new_path)) + return None + + +def render_chain(adapter: Any, path: list[dict]) -> str: + steps = [] + for edge in path: + rendered = adapter.render_edge(edge).rstrip(".") + steps.append(rendered[0].lower() + rendered[1:]) + return "; ".join(steps) diff --git a/root/wander-train/stage5m2c_wikidata_generalist_federation.py b/root/wander-train/stage5m2c_wikidata_generalist_federation.py new file mode 100644 index 0000000000000000000000000000000000000000..f87fbfad83547b3d09d076f69f7ad4c77152a698 --- /dev/null +++ b/root/wander-train/stage5m2c_wikidata_generalist_federation.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""M2c authenticated minimal generalist federation from Wikidata CC0 data. + +The acquisition layer resolves a predeclared domain term list through the +official Wikidata API, stores every raw response, records entity revisions and +JSON pointers, deterministically lowers labels/descriptions/type statements to +M2b source prose, then compiles the unchanged frozen-carrier HRR specialists. +The normalized prose is never treated as an independent authority: the +federation manifest binds each generated sentence to its raw snapshot. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +import urllib.parse +import urllib.request +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +from oracle_10d_resolution_state import canonical_sha256 +from stage5m2a_relational_corpus_ir import SCHEMA as CORPUS_SCHEMA +from stage5m2b_decimal_sentence_successor import compile_decimal_safe_corpus + + +SCHEMA = "oracle-stage5m2c-wikidata-generalist-federation-v1" +API = "https://www.wikidata.org/w/api.php" +LICENSE_NAME = "Creative Commons CC0 1.0" +LICENSE_URL = "https://creativecommons.org/publicdomain/zero/1.0/" +DATA_ACCESS_URL = "https://www.wikidata.org/wiki/Help:Data_access" +COPYRIGHT_URL = "https://www.wikidata.org/wiki/Wikidata:Copyright" +USER_AGENT = "Oracle-HRR-M2c-Generalist-Federation/1.0 (research; prometheus7.com)" + +DOMAIN_TERMS: Mapping[str, tuple[str, ...]] = { + "language": ("language", "English", "Spanish", "linguistics", "grammar"), + "mathematics": ("mathematics", "algebra", "geometry", "calculus", "number theory"), + "physical_science": ("physics", "chemistry", "thermodynamics", "quantum mechanics", "electromagnetism"), + "life_science": ("biology", "ecology", "genetics", "cell biology", "evolution"), + "earth_space_science": ("Earth science", "astronomy", "planetary science", "geology", "meteorology"), + "computing": ("computer science", "algorithm", "operating system", "database", "computer network"), + "engineering": ("engineering", "mechanical engineering", "electrical engineering", "civil engineering", "systems engineering"), + "history_civics": ("history", "government", "democracy", "constitution", "citizenship"), + "economics": ("economics", "market", "inflation", "gross domestic product", "monetary policy"), + "everyday_procedural": ("cooking", "first aid", "navigation", "hygiene", "food preservation"), +} + + +def _sha_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _sha(path: Path) -> str: + return _sha_bytes(path.read_bytes()) + + +def _fetch(params: Mapping[str, str], *, timeout: float = 30.0) -> tuple[bytes, str]: + query = urllib.parse.urlencode(params) + url = API + "?" + query + request = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=timeout) as response: + resolved = response.geturl() + parsed = urllib.parse.urlparse(resolved) + if parsed.scheme != "https" or (parsed.hostname or "").casefold() != "www.wikidata.org": + raise PermissionError("Wikidata redirect escaped the allowlisted HTTPS host") + payload = response.read(4_000_001) + if len(payload) > 4_000_000: + raise ValueError("Wikidata response exceeded acquisition limit") + json.loads(payload.decode("utf-8")) + return payload, resolved + + +def _store_raw(root: Path, stem: str, payload: bytes, url: str) -> dict[str, Any]: + digest = _sha_bytes(payload) + path = root / f"{stem}_{digest[:16]}.json" + path.write_bytes(payload) + return {"path": str(path), "sha256": digest, "bytes": len(payload), "resolved_url": url} + + +def _english(entity: Mapping[str, Any], field: str) -> str | None: + value = entity.get(field, {}).get("en") + return str(value.get("value", "")).strip() if isinstance(value, dict) and value.get("value") else None + + +def _entity_targets(entity: Mapping[str, Any], property_id: str, maximum: int = 2) -> tuple[str, ...]: + result = [] + for statement in entity.get("claims", {}).get(property_id, ()): + if statement.get("rank") == "deprecated": + continue + value = statement.get("mainsnak", {}).get("datavalue", {}).get("value") + target = value.get("id") if isinstance(value, dict) else None + if isinstance(target, str) and target.startswith("Q") and target not in result: + result.append(target) + if len(result) >= maximum: + break + return tuple(result) + + +def _chunks(values: Sequence[str], size: int = 45): + for start in range(0, len(values), size): + yield values[start:start + size] + + +@dataclass(frozen=True) +class FederationResult: + path: Path + payload: Mapping[str, Any] + file_sha256: str + + @classmethod + def load(cls, path: Path) -> "FederationResult": + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schema_version") != SCHEMA: + raise ValueError("unsupported M2c federation") + if canonical_sha256(payload["semantic_identity"]) != payload["semantic_identity_sha256"]: + raise ValueError("M2c semantic identity changed") + for artifact in payload["artifacts"]: + target = Path(artifact["path"]) + if not target.is_file() or _sha(target) != artifact["sha256"]: + raise ValueError(f"M2c artifact changed: {target}") + return cls(path.resolve(), payload, _sha(path)) + + +def acquire_and_compile( + output_root: Path, + *, + carrier_sha256: str, +) -> FederationResult: + started = time.perf_counter(); output_root = output_root.resolve() + output_root.mkdir(parents=True, exist_ok=False) + raw_root = output_root / "raw_wikidata"; raw_root.mkdir() + normalized_root = output_root / "normalized_corpora"; normalized_root.mkdir() + artifacts: list[dict[str, Any]] = [] + resolutions: list[dict[str, Any]] = [] + + for domain, terms in DOMAIN_TERMS.items(): + for index, term in enumerate(terms): + payload, url = _fetch({"action": "wbsearchentities", "search": term, "language": "en", "uselang": "en", "format": "json", "limit": "10", "type": "item"}) + artifact = _store_raw(raw_root, f"search_{domain}_{index}", payload, url); artifacts.append({"kind": "wikidata_search_snapshot", **artifact}) + decoded = json.loads(payload.decode("utf-8")); results = decoded.get("search") or [] + exact = [value for value in results if str(value.get("label", "")).casefold() == term.casefold()] + selected = (exact or results)[0] if results else None + if not isinstance(selected, dict) or not str(selected.get("id", "")).startswith("Q"): + raise ValueError(f"Wikidata term did not resolve: {domain}:{term}") + resolutions.append({"domain": domain, "term": term, "qid": selected["id"], "search_label": selected.get("label"), "search_description": selected.get("description"), "exact_label_match": bool(exact), "search_snapshot_sha256": artifact["sha256"]}) + + qids = sorted({value["qid"] for value in resolutions}) + entities: dict[str, Any] = {} + entity_snapshot_by_qid: dict[str, str] = {} + for batch_index, batch in enumerate(_chunks(qids)): + payload, url = _fetch({"action": "wbgetentities", "ids": "|".join(batch), "props": "labels|descriptions|claims|info", "languages": "en", "format": "json"}) + artifact = _store_raw(raw_root, f"entities_primary_{batch_index}", payload, url); artifacts.append({"kind": "wikidata_entity_snapshot", **artifact}) + values = json.loads(payload.decode("utf-8")).get("entities", {}) + entities.update(values) + for qid in values: entity_snapshot_by_qid[qid] = artifact["sha256"] + + target_qids = sorted({target for qid in qids for prop in ("P31", "P279") for target in _entity_targets(entities[qid], prop)}) + for batch_index, batch in enumerate(_chunks(target_qids)): + payload, url = _fetch({"action": "wbgetentities", "ids": "|".join(batch), "props": "labels|descriptions|info", "languages": "en", "format": "json"}) + artifact = _store_raw(raw_root, f"entities_targets_{batch_index}", payload, url); artifacts.append({"kind": "wikidata_entity_snapshot", **artifact}) + values = json.loads(payload.decode("utf-8")).get("entities", {}) + entities.update(values) + for qid in values: entity_snapshot_by_qid[qid] = artifact["sha256"] + + mappings: list[dict[str, Any]] = [] + documents = [] + by_domain: dict[str, list[dict[str, Any]]] = defaultdict(list) + for resolution in resolutions: + by_domain[resolution["domain"]].append(resolution) + for domain in DOMAIN_TERMS: + lines = [] + for resolution in by_domain[domain]: + qid = resolution["qid"]; entity = entities[qid] + label = _english(entity, "labels") or str(resolution["search_label"]) + description = _english(entity, "descriptions") or str(resolution.get("search_description") or "") + revision = int(entity.get("lastrevid") or 0) + if description: + sentence = f"{label} is defined as {description} [Wikidata {qid} revision {revision}]." + lines.append(sentence) + mappings.append({"domain": domain, "qid": qid, "revision": revision, "sentence": sentence, "sentence_sha256": _sha_bytes(sentence.encode("utf-8")), "source_snapshot_sha256": entity_snapshot_by_qid[qid], "json_pointer": f"/entities/{qid}/descriptions/en/value", "transform": "label_is_defined_as_description"}) + for property_id, predicate in (("P31", "instance_of"), ("P279", "subclass_of")): + for target in _entity_targets(entity, property_id): + target_label = _english(entities.get(target, {}), "labels") + if not target_label: + continue + sentence = f"{label} is a {target_label} [Wikidata {qid} revision {revision}]." + lines.append(sentence) + mappings.append({"domain": domain, "qid": qid, "revision": revision, "sentence": sentence, "sentence_sha256": _sha_bytes(sentence.encode("utf-8")), "source_snapshot_sha256": entity_snapshot_by_qid[qid], "json_pointer": f"/entities/{qid}/claims/{property_id}", "target_qid": target, "target_snapshot_sha256": entity_snapshot_by_qid.get(target), "transform": predicate}) + path = normalized_root / f"{domain}.txt" + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + artifacts.append({"kind": "normalized_domain_corpus", "path": str(path), "sha256": _sha(path)}) + documents.append({"expert_id": f"generalist_{domain}", "source_document_id": f"wikidata_{domain}", "document_path": f"normalized_corpora/{path.name}", "source_url": DATA_ACCESS_URL, "source_split": "m2c_cc0_snapshot", "competencies": [domain, "wikidata_definitions", "wikidata_taxonomy"]}) + + corpus_spec = output_root / "corpus_spec.json" + corpus_spec.write_text(json.dumps({"schema_version": CORPUS_SCHEMA, "documents": documents}, indent=2, sort_keys=True) + "\n", encoding="utf-8") + artifacts.append({"kind": "corpus_spec", "path": str(corpus_spec), "sha256": _sha(corpus_spec)}) + compiler_root = output_root / "compiled" + bundle_path = output_root / "m2c_specialist_bundle.json" + compilation = compile_decimal_safe_corpus(corpus_spec, carrier_sha256=carrier_sha256, output_dir=compiler_root, bundle_path=bundle_path) + artifacts.append({"kind": "compiled_specialist_bundle", "path": str(bundle_path), "sha256": _sha(bundle_path)}) + + admitted_sentence_hashes = {_sha_bytes(value["evidence"].encode("utf-8")) for value in compilation.payload["claims"]} + for mapping in mappings: + mapping["admitted_by_compiler"] = mapping["sentence_sha256"] in admitted_sentence_hashes + mapping_path = output_root / "source_to_sentence_map.jsonl" + mapping_path.write_text("".join(json.dumps(value, sort_keys=True) + "\n" for value in mappings), encoding="utf-8") + artifacts.append({"kind": "source_to_sentence_map", "path": str(mapping_path), "sha256": _sha(mapping_path)}) + resolution_path = output_root / "term_resolution.jsonl" + resolution_path.write_text("".join(json.dumps(value, sort_keys=True) + "\n" for value in resolutions), encoding="utf-8") + artifacts.append({"kind": "term_resolution", "path": str(resolution_path), "sha256": _sha(resolution_path)}) + + per_domain = {} + for domain in DOMAIN_TERMS: + claims = [value for value in compilation.payload["claims"] if value["expert_id"] == f"generalist_{domain}"] + per_domain[domain] = {"requested_terms": len(DOMAIN_TERMS[domain]), "resolved_terms": len(by_domain[domain]), "exact_label_matches": sum(value["exact_label_match"] for value in by_domain[domain]), "normalized_sentences": sum(value["domain"] == domain for value in mappings), "admitted_claims": len(claims), "quarantined_sentences": sum(value.get("expert_id") == f"generalist_{domain}" for value in compilation.payload["quarantine_ledger"])} + semantic_identity = {"carrier_sha256": carrier_sha256, "compiler_bundle_sha256": _sha(bundle_path), "raw_snapshot_sha256": sorted(value["sha256"] for value in artifacts if value["kind"].startswith("wikidata_")), "normalized_corpus_sha256": {domain: _sha(normalized_root / f"{domain}.txt") for domain in DOMAIN_TERMS}, "source_mapping_sha256": _sha(mapping_path)} + manifest_path = output_root / "federation_manifest.json" + payload = {"schema_version": SCHEMA, "status": "compiled", "claim_boundary": "Minimal ten-domain federation of Wikidata English descriptions and P31/P279 type edges lowered through the frozen M2b grammar; breadth is shallow and structured retrieval is not yet natural-language realization.", "authority": "Wikidata", "license_name": LICENSE_NAME, "license_url": LICENSE_URL, "data_access_url": DATA_ACCESS_URL, "copyright_url": COPYRIGHT_URL, "training_evaluation_separation": "Wikidata CC0 snapshots are training sources; the M0 BIG-bench commit and cases are not ingested.", "domains": per_domain, "term_resolutions": resolutions, "source_mapping_path": str(mapping_path), "compiler_bundle": {"path": str(bundle_path), "sha256": _sha(bundle_path), "claims": compilation.payload["admitted_claim_count"], "checkpoints": len(compilation.payload["checkpoints"])}, "artifacts": artifacts, "performance": {"total_wall_seconds": time.perf_counter() - started, "compiler_wall_seconds": compilation.payload["performance"]["total_wall_seconds"]}, "semantic_identity": semantic_identity, "semantic_identity_sha256": canonical_sha256(semantic_identity)} + manifest_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return FederationResult.load(manifest_path) + + +def main() -> None: + parser = argparse.ArgumentParser(); parser.add_argument("--output-root", type=Path, required=True); parser.add_argument("--carrier-sha256", required=True); args = parser.parse_args() + result = acquire_and_compile(args.output_root, carrier_sha256=args.carrier_sha256) + print(json.dumps({"manifest": str(result.path), "sha256": result.file_sha256, "claims": result.payload["compiler_bundle"]["claims"], "domains": len(result.payload["domains"]), "seconds": result.payload["performance"]["total_wall_seconds"]}, indent=2)) + + +if __name__ == "__main__": main() diff --git a/root/wander-train/stage5m3a_evidence_addressed_realizer.py b/root/wander-train/stage5m3a_evidence_addressed_realizer.py new file mode 100644 index 0000000000000000000000000000000000000000..1b7780bd88164526ac1e51cf8867764ec0bf2e9a --- /dev/null +++ b/root/wander-train/stage5m3a_evidence_addressed_realizer.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""M3a evidence-addressed realization over authenticated specialist graphs. + +The realizer generalizes L5's hierarchy without importing its self-architecture +claim constructor. It selects a bounded authenticated subgraph, assigns every +surface clause an authority class, transports opaque clause handles through the +4:1 protected Tree spine, and chooses a whole-answer layout only after terminal +cleanup. It is deterministic and contains no token-prediction model. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import time +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np + +from oracle_10d_resolution_state import canonical_sha256 +from stage5m2b_decimal_sentence_successor import DecimalSafeCompilation +from stage5m2c3_fragment_authority_successor import FragmentAuthorizedFederation +from train_stage2a_role_ladder import HRR_DIM, PATHS, SEPHIROTH, bind, cosine, normalize, unbind, unitary + + +SCHEMA = "oracle-stage5m3a-evidence-addressed-realizer-v1" +ARTIFACTS = {"word", "equation", "sentence", "paragraph", "essay", "paper"} +MODES = {"define", "explain", "summarize", "compare"} + + +def _digest(value: Any) -> str: + return canonical_sha256(value) + + +def _surface_id(value: str) -> str: + return " ".join(str(value).replace("_", " ").split()) + + +def _sentence_case(value: str) -> str: + value = " ".join(value.split()) + return value[:1].upper() + value[1:] if value else value + + +def _article(value: str) -> str: + return "an" if value[:1].casefold() in "aeiou" else "a" + + +@dataclass(frozen=True) +class RealizationRequest: + domain: str + subjects: tuple[str, ...] + artifact: str = "paragraph" + mode: str = "explain" + paragraphs: int = 3 + predicate: str | None = None + world_id: str | None = None + temporary_premises: tuple[Mapping[str, Any], ...] = () + + def __post_init__(self) -> None: + if self.artifact not in ARTIFACTS: + raise ValueError("unsupported M3a artifact") + if self.mode not in MODES: + raise ValueError("unsupported M3a mode") + if not self.subjects or len(self.subjects) > 2: + raise ValueError("M3a requires one subject, or two for comparison") + if self.mode == "compare" and len(self.subjects) != 2: + raise ValueError("comparison requires exactly two subjects") + if self.temporary_premises and not self.world_id: + raise ValueError("temporary premises require an explicit world id") + + def public(self) -> dict[str, Any]: + return { + "domain": self.domain, "subjects": list(self.subjects), "artifact": self.artifact, + "mode": self.mode, "paragraphs": self.paragraphs, "predicate": self.predicate, + "world_id": self.world_id, "temporary_premises": [dict(value) for value in self.temporary_premises], + } + + +@dataclass(frozen=True) +class SurfaceClause: + clause_id: str + text: str + authority_class: str + claim_ids: tuple[str, ...] = () + proof_path: tuple[str, ...] = () + world_id: str | None = None + inline_citations: tuple[str, ...] = () + + def public(self) -> dict[str, Any]: + return { + "clause_id": self.clause_id, "text": self.text, + "authority_class": self.authority_class, "claim_ids": list(self.claim_ids), + "proof_path": list(self.proof_path), "world_id": self.world_id, + "inline_citations": list(self.inline_citations), + } + + +class EvidenceAddressedRealizer: + def __init__(self, federation_manifest: Path): + self.federation = FragmentAuthorizedFederation.load(federation_manifest.resolve()) + self.bundle = DecimalSafeCompilation.load(Path(self.federation.payload["compiler_bundle"]["path"])) + self.claims = {value["claim_id"]: value for value in self.bundle.payload["claims"]} + self.by_domain_subject: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for claim in self.claims.values(): + domain = claim["expert_id"].removeprefix("generalist_") + self.by_domain_subject[(domain, claim["semantic"]["subject"])].append(claim) + + def _subgraph(self, request: RealizationRequest) -> tuple[dict[str, Any], ...]: + limit = {"word": 1, "equation": 1, "sentence": 1, "paragraph": 4, "essay": 9, "paper": 12}[request.artifact] + selected: list[dict[str, Any]] = [] + seen = set() + if request.mode == "compare": + quota = max(1, limit // len(request.subjects)) + for subject in request.subjects: + candidates = sorted( + self.by_domain_subject.get((request.domain, subject), ()), + key=lambda value: (value["semantic"]["predicate"] != "defined_as", value["semantic_key"], value["claim_id"]), + ) + if request.predicate: + candidates = [value for value in candidates if value["semantic"]["predicate"] == request.predicate] + for claim in candidates[:quota]: + selected.append(claim); seen.add(claim["claim_id"]) + return tuple(selected[:limit]) + frontier = [(subject, 0) for subject in request.subjects] + while frontier and len(selected) < limit: + subject, depth = frontier.pop(0) + candidates = sorted(self.by_domain_subject.get((request.domain, subject), ()), key=lambda value: (value["semantic"]["predicate"] != "defined_as", value["semantic_key"], value["claim_id"])) + if request.predicate: + candidates = [value for value in candidates if value["semantic"]["predicate"] == request.predicate] + if request.artifact == "equation": + candidates = [value for value in candidates if value["semantic"]["claim_type"] == "quantity"] + elif request.artifact == "word": + candidates = [value for value in candidates if len(value["object_surface"].split()) == 1] + for claim in candidates: + if claim["claim_id"] in seen: + continue + selected.append(claim); seen.add(claim["claim_id"]) + if len(selected) >= limit: + break + destination = claim["semantic"]["object"] + if depth == 0 and (request.domain, destination) in self.by_domain_subject: + frontier.append((destination, 1)) + return tuple(selected) + + @staticmethod + def _fact_surface(claim: Mapping[str, Any], citation: str) -> str: + semantic = claim["semantic"]; subject = _sentence_case(claim["subject_surface"]); obj = claim["object_surface"] + predicate = semantic["predicate"]; polarity = semantic["polarity"] + if predicate == "defined_as": + core = f"{subject} is defined in the authenticated source as {obj}" + elif predicate in {"instance_of", "subclass_of"}: + relation = "is not classified as" if polarity == "negative" else "is classified as" + core = f"{subject} {relation} {_article(obj)} {obj}" + elif predicate.startswith("has_"): + attribute = _surface_id(predicate.removeprefix("has_")) + core = f"{subject} has {attribute} {obj}" + else: + relation = _surface_id(predicate) + core = f"{subject} {'does not ' if polarity == 'negative' else ''}{relation} {obj}" + qualifiers = [] + if semantic["conditions"]: + qualifiers.append("under the condition that " + "; ".join(semantic["conditions"])) + if semantic["temporal"]: + qualifiers.append("within " + json.dumps(semantic["temporal"], sort_keys=True)) + if qualifiers: + core += ", " + ", and ".join(qualifiers) + return core.rstrip(". ") + f" [{citation}]." + + @staticmethod + def _reference(claim: Mapping[str, Any], citation: str) -> dict[str, Any]: + return { + "citation_id": citation, "claim_id": claim["claim_id"], + "evidence": claim["evidence"], "source_document_id": claim["source_document_id"], + "source_document_sha256": claim["source_document_sha256"], "source_url": claim["source_url"], + "provenance_sha256": claim["provenance_sha256"], "source_citations": claim["citations"], + } + + @staticmethod + def _spine(clauses: Sequence[SurfaceClause], objective_sha: str) -> dict[str, Any]: + handles = {value.clause_id: unitary("m3a:clause:" + _digest(value.public())) for value in clauses} + roles = {value.clause_id: unitary("m3a:clause-role:" + value.clause_id) for value in clauses} + payload = normalize(sum((bind(roles[value.clause_id], handles[value.clause_id]) for value in clauses), start=np.zeros(HRR_DIM, dtype=np.complex64))) + root = unitary("m3a:objective-root:" + objective_sha) + residual = payload - np.complex64(np.vdot(root, payload)) * root + residual = normalize(residual) + cosines = [] + for node in SEPHIROTH: + local = unitary("m3a:tree-node:" + node + ":" + objective_sha) + local = local - np.complex64(np.vdot(root, local)) * root + residual = normalize(residual + np.complex64(0.04) * normalize(local)) + protected = normalize(np.float32(4.0) * root + residual) + cosines.append(cosine(protected, root)) + recoveries = [] + for clause in clauses: + recovered = unbind(payload, roles[clause.clause_id]) + scores = {key: cosine(recovered, value) for key, value in handles.items()} + winner = max(scores, key=lambda key: (scores[key], key)); ordered = sorted(scores.values(), reverse=True) + recoveries.append({"clause_id": clause.clause_id, "winner": winner, "exact": winner == clause.clause_id, "margin": ordered[0] - (ordered[1] if len(ordered) > 1 else 0.0)}) + return { + "protected_root_ratio": "4:1", "route": list(SEPHIROTH), "tree_nodes_registered": len(SEPHIROTH), + "tree_paths_registered": len(PATHS), "pre_malkuth_payload_unbinds": 0, + "malkuth_payload_unbinds": len(clauses), "minimum_protected_root_cosine": min(cosines), + "terminal_cleanup_exact": all(value["exact"] for value in recoveries), + "minimum_cleanup_margin": min(value["margin"] for value in recoveries), "recoveries": recoveries, + } + + @staticmethod + def _layout(paragraphs: Sequence[Sequence[SurfaceClause]], request: RealizationRequest) -> tuple[str, dict[str, Any]]: + canonical_paragraphs = [" ".join(value.text for value in paragraph) for paragraph in paragraphs if paragraph] + canonical = "\n\n".join(canonical_paragraphs) + collapsed = " ".join(canonical_paragraphs) + reversed_text = "\n\n".join(reversed(canonical_paragraphs)) + uncited = __import__("re").sub(r"\s*\[C\d+\]", "", canonical) + losses = { + "canonical": 0.0, + "collapsed": 0.0 if request.artifact in {"word", "equation", "sentence", "paragraph"} else 1.5, + "reversed": 2.5, + "uncited": 4.0, + } + candidates = (("canonical", canonical), ("collapsed", collapsed), ("reversed", reversed_text), ("uncited", uncited)) + selected_index = min(range(len(candidates)), key=lambda index: (losses[candidates[index][0]], candidates[index][0])) + penalty = 5.0; matrix = np.full((len(candidates), len(candidates)), penalty, dtype=np.float64) + for index, (name, _text) in enumerate(candidates): matrix[index, index] = losses[name] - penalty + assignment = [int(index == selected_index) for index in range(len(candidates))] + return candidates[selected_index][1], { + "selected_layout": candidates[selected_index][0], "candidate_layouts": [value[0] for value in candidates], + "candidate_losses": [losses[value[0]] for value in candidates], + "candidate_text_sha256s": [hashlib.sha256(value[1].encode("utf-8")).hexdigest() for value in candidates], + "quadratic_matrix_sha256": hashlib.sha256(np.ascontiguousarray(matrix).tobytes()).hexdigest(), + "assignment": assignment, "assignment_hamming_weight": sum(assignment), + "exhaustive_one_hot_minimum_verified": selected_index == int(np.argmin(np.diag(matrix))), + "whole_document_selected_globally": True, "next_token_predictions": 0, + } + + @staticmethod + def _partition(clauses: Sequence[SurfaceClause], request: RealizationRequest) -> list[list[SurfaceClause]]: + if request.artifact in {"word", "equation", "sentence", "paragraph"}: + return [list(clauses)] + count = min(max(3, int(request.paragraphs)), 6) + groups = [[] for _ in range(count)] + for index, clause in enumerate(clauses): + groups[min(count - 1, math.floor(index * count / max(1, len(clauses))))].append(clause) + return groups + + def realize(self, request: RealizationRequest) -> dict[str, Any]: + started = time.perf_counter(); request_sha = _digest(request.public()); claims = self._subgraph(request) + if not claims and not request.temporary_premises: + return {"schema_version": SCHEMA, "status": "withheld", "reason": "no_authenticated_topic_subgraph", "request": request.public(), "request_sha256": request_sha, "response": "I need an authenticated specialist relation for this topic before I can construct the requested answer.", "factual_release": False, "clauses": [], "references": [], "latency_ms": (time.perf_counter() - started) * 1000.0} + if request.artifact in {"essay", "paper"} and len(claims) + len(request.temporary_premises) < 3: + return {"schema_version": SCHEMA, "status": "withheld", "reason": "topic_graph_too_small_for_long_form", "request": request.public(), "request_sha256": request_sha, "response": "The authenticated topic graph is too small for a non-redundant long-form answer.", "factual_release": False, "clauses": [], "references": [], "latency_ms": (time.perf_counter() - started) * 1000.0} + + references = []; clauses: list[SurfaceClause] = [] + if request.artifact not in {"word", "equation", "sentence"}: + claim_ids = tuple(value["claim_id"] for value in claims) + subjects = " and ".join(_surface_id(value) for value in request.subjects) + clauses.append(SurfaceClause("synthesis_intro", f"This account examines {subjects} through {len(claims)} authenticated relations.", "inferred", claim_ids=claim_ids, proof_path=("count_authenticated_subgraph", *claim_ids))) + for index, claim in enumerate(claims, start=1): + citation = f"C{index}"; references.append(self._reference(claim, citation)) + if request.artifact == "word": + text = claim["object_surface"] + elif request.artifact == "equation" and claim["semantic"]["claim_type"] == "quantity": + text = f"{claim['semantic']['subject']}.{claim['semantic']['predicate']} = {claim['object_surface']}" + else: + text = self._fact_surface(claim, citation) + clauses.append(SurfaceClause(f"fact_{index:02d}", text, "authenticated_fact", claim_ids=(claim["claim_id"],), proof_path=("authenticated_claim", claim["claim_id"], claim["provenance_sha256"]), inline_citations=(() if request.artifact in {"word", "equation"} else (citation,)))) + for index, premise in enumerate(request.temporary_premises, start=1): + subject = _sentence_case(str(premise["subject_surface"])); predicate = _surface_id(str(premise["predicate"])); obj = str(premise["object_surface"]) + text = f"Within the temporary world {request.world_id}, {subject} {predicate} {obj}." + clauses.append(SurfaceClause(f"premise_{index:02d}", text, "hypothetical", proof_path=("temporary_premise", _digest(dict(premise))), world_id=request.world_id)) + if request.artifact not in {"word", "equation", "sentence"}: + ids = tuple(value["claim_id"] for value in claims) + boundary = "These relations support the bounded description above; they do not license claims beyond the cited graph." + clauses.append(SurfaceClause("synthesis_boundary", boundary, "inferred", claim_ids=ids, proof_path=("bounded_graph_closure", *ids))) + + paragraphs = self._partition(clauses, request) + response, inverse = self._layout(paragraphs, request) + if request.artifact == "paper": + response = "## Evidence-addressed analysis\n\n" + response + "\n\n## References\n\n" + "\n".join(f"[{value['citation_id']}] {value['source_url']} — {value['source_document_id']} ({value['source_document_sha256'][:12]})." for value in references) + spine = self._spine(clauses, request_sha) + factual = [value for value in clauses if value.authority_class == "authenticated_fact"] + citation_complete = all(value.inline_citations for value in factual) if request.artifact not in {"word", "equation"} else all(value.claim_ids for value in factual) + artifact = { + "schema_version": SCHEMA, "status": "answered", "reason": "authenticated_subgraph_realized", + "request": request.public(), "request_sha256": request_sha, "response": response, + "factual_release": bool(factual), "clauses": [value.public() for value in clauses], + "references": references, "claim_graph": {"selected_claim_ids": [value["claim_id"] for value in claims], "selected_claim_count": len(claims), "source_federation_sha256": self.federation.file_sha256}, + "paragraph_plan": [[value.clause_id for value in paragraph] for paragraph in paragraphs], + "tree_spine": spine, "whole_answer_inverse": inverse, + "audit": { + "all_factual_clauses_evidence_addressed": all(value.claim_ids for value in factual), + "all_inferred_clauses_have_proof_paths": all(value.proof_path for value in clauses if value.authority_class == "inferred"), + "all_hypothetical_clauses_world_scoped": all(value.world_id for value in clauses if value.authority_class == "hypothetical"), + "citation_complete": citation_complete, + "duplicate_clause_count": len(clauses) - len({value.text for value in clauses}), + "traditional_language_model_calls": 0, "transformer_calls": 0, + "pretrained_embedding_encoder_calls": 0, "next_token_predictions": 0, "gradient_steps": 0, + }, + "latency_ms": (time.perf_counter() - started) * 1000.0, + } + artifact["artifact_sha256"] = _digest({key: value for key, value in artifact.items() if key not in {"latency_ms", "artifact_sha256"}}) + return artifact + + +__all__ = ["SCHEMA", "RealizationRequest", "EvidenceAddressedRealizer"] diff --git a/root/wander-train/stage5m3b_conversational_federation.py b/root/wander-train/stage5m3b_conversational_federation.py new file mode 100644 index 0000000000000000000000000000000000000000..1181730f6984ab0b806e034fa2762ab615548d92 --- /dev/null +++ b/root/wander-train/stage5m3b_conversational_federation.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""M3b natural-language and discourse bridge to the frozen M3a realizer.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import time +import unicodedata +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from oracle_10d_resolution_state import canonical_sha256 +from stage5m1d_language_context_successor import compile_act +from stage5m3a_evidence_addressed_realizer import EvidenceAddressedRealizer, RealizationRequest + + +SCHEMA = "oracle-stage5m3b-conversational-federation-v1" +IDENTITY = { + "name": "Oracle Relational V3", + "kind": "evidence-addressed relational cognitive runtime", + "purpose": "compile language into bounded relational programs and realize only authenticated or explicitly world-scoped claims", + "commitments": [ + "preserve the distinction between language structure and factual authority", + "cite authenticated factual clauses", + "mark derivations and hypothetical worlds", + "withhold rather than force an unknown address to a nearby stored value", + ], +} +NUMBER_WORDS = {"three": 3, "four": 4, "five": 5, "six": 6} +CONCEPT_ALIASES = { + "geometria": "geometry", "geometrie": "geometry", + "physique": "physics", "fisica": "physics", "physik": "physics", + "biologia": "biology", "biologie": "biology", + "matematicas": "mathematics", "mathematiques": "mathematics", "mathematik": "mathematics", + "informatica": "computer_science", "informatique": "computer_science", "informatik": "computer_science", + "economia": "economics", "economie": "economics", + "historia": "history", "histoire": "history", "geschichte": "history", + "quimica": "chemistry", "chimie": "chemistry", "chemie": "chemistry", +} +ARTICLE = re.compile(r"^(?:the|a|an|el|la|los|las|le|les|un|une|der|die|das|ein|eine)\s+", re.I) +DOCUMENT = re.compile( + r"^\s*(?:please\s+)?(?:write|compose|draft|produce|create|give\s+me)\s+" + r"(?:a\s+|an\s+)?(?:(?P[3-6]|three|four|five|six)[-\s]+paragraph\s+)?" + r"(?Pessay|paper|paragraph|sentence)\s+(?:about|on|explaining|concerning)\s+(?P.+?)\s*[?.!]*$", + re.I, +) +QUESTION = ( + (re.compile(r"^\s*(?:what|who)\s+(?:is|are)\s+(.+?)\s*[?.!]*$", re.I), "define"), + (re.compile(r"^\s*(?:tell\s+me\s+about|describe)\s+(.+?)\s*[?.!]*$", re.I), "explain"), +) +FOLLOW_MORE = re.compile(r"^\s*(?:tell\s+me\s+more|go\s+on|continue|expand(?:\s+that)?|say\s+more)\s*[?.!]*$", re.I) +FOLLOW_ARTIFACT = re.compile(r"^\s*(?:write|turn|make)\s+(?:that|it)\s+(?:as|into)\s+(?:a|an)\s+(essay|paper|paragraph|sentence)\s*[?.!]*$", re.I) +FOLLOW_COMPARE = re.compile(r"^\s*compare\s+(?:that|it)\s+(?:with|to|and)\s+(.+?)\s*[?.!]*$", re.I) +FOLLOW_WHAT = re.compile(r"^\s*what\s+about\s+(.+?)\s*[?.!]*$", re.I) + + +def _ascii(value: str) -> str: + normalized = unicodedata.normalize("NFKD", str(value)) + return "".join(character for character in normalized if not unicodedata.combining(character)) + + +def _key(value: str) -> str: + text = ARTICLE.sub("", _ascii(str(value)).casefold().strip()) + text = re.sub(r"\b(?:briefly|please|for me|in detail)\b", " ", text) + return re.sub(r"_+", "_", re.sub(r"[^a-z0-9]+", "_", text)).strip("_") + + +class HashChainedSessionStore: + def __init__(self, root: Path): + self.root = root.resolve(); self.root.mkdir(parents=True, exist_ok=True) + + def _path(self, session_id: str) -> Path: + return self.root / (hashlib.sha256(session_id.encode("utf-8")).hexdigest() + ".json") + + def load(self, session_id: str) -> dict[str, Any]: + path = self._path(session_id) + if not path.exists(): + return {"session_id_sha256": hashlib.sha256(session_id.encode()).hexdigest(), "turns": [], "last": None} + payload = json.loads(path.read_text(encoding="utf-8")); prior = "0" * 64 + for turn in payload["turns"]: + if turn["previous_sha256"] != prior: + raise ValueError("M3b session chain predecessor changed") + body = {key: value for key, value in turn.items() if key != "turn_sha256"} + if canonical_sha256(body) != turn["turn_sha256"]: + raise ValueError("M3b session turn changed") + prior = turn["turn_sha256"] + return payload + + def append(self, session_id: str, *, text: str, program: Mapping[str, Any], result: Mapping[str, Any], state: Mapping[str, Any] | None) -> dict[str, Any]: + payload = self.load(session_id); prior = payload["turns"][-1]["turn_sha256"] if payload["turns"] else "0" * 64 + body = {"ordinal": len(payload["turns"]), "previous_sha256": prior, "input_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), "program_sha256": canonical_sha256(program), "result_sha256": canonical_sha256({key: value for key, value in result.items() if key != "latency_ms"}), "state": dict(state or {})} + turn = {**body, "turn_sha256": canonical_sha256(body)}; payload["turns"].append(turn); payload["last"] = dict(state) if state else payload.get("last") + path = self._path(session_id); temporary = path.with_suffix(".tmp"); temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"); os.replace(temporary, path) + return {"turns": len(payload["turns"]), "head_sha256": turn["turn_sha256"], "hash_chain_valid": True, "path": str(path)} + + +@dataclass(frozen=True) +class CompiledConversationAct: + operator: str + topic_surfaces: tuple[str, ...] + artifact: str + paragraphs: int + surface_language: str + source: str + condition: str | None = None + + def public(self) -> dict[str, Any]: + value = dict(self.__dict__); value["topic_surfaces"] = list(self.topic_surfaces); value["program_sha256"] = canonical_sha256(value); value["structural_authority"] = True; value["factual_authority"] = False; return value + + +class ConversationalFederationOracle: + def __init__(self, federation_manifest: Path, *, state_root: Path): + self.realizer = EvidenceAddressedRealizer(federation_manifest) + self.sessions = HashChainedSessionStore(state_root / "m3b_sessions") + self.identity = {**IDENTITY, "identity_sha256": canonical_sha256(IDENTITY)} + self.subject_domains: dict[str, set[str]] = {} + for domain, subject in self.realizer.by_domain_subject: + self.subject_domains.setdefault(subject, set()).add(domain) + self.subject_keys = sorted(self.subject_domains, key=lambda value: (-len(value), value)) + + def _resolve_surface(self, surface: str) -> tuple[str, str] | None: + key = CONCEPT_ALIASES.get(_key(surface), _key(surface)) + matches = [] + if key in self.subject_domains: + matches.append(key) + else: + padded = "_" + key + "_" + matches.extend(subject for subject in self.subject_keys if "_" + subject + "_" in padded) + matches = list(dict.fromkeys(matches)) + if len(matches) != 1: + return None + domains = sorted(self.subject_domains[matches[0]]) + return (domains[0], matches[0]) if len(domains) == 1 else None + + @staticmethod + def _artifact_for(operator: str) -> str: + return "sentence" if operator in {"define", "classify"} else "paragraph" + + def _compile(self, text: str, session_id: str) -> tuple[CompiledConversationAct | None, str | None]: + prior = self.sessions.load(session_id).get("last") + match = DOCUMENT.match(text) + if match: + count = match.group("count"); paragraphs = int(count) if count and count.isdigit() else NUMBER_WORDS.get((count or "").casefold(), 3) + artifact = match.group("artifact").casefold() + return CompiledConversationAct("explain", (match.group("topic"),), artifact, paragraphs, "en", "m3b_document_construction"), None + match = FOLLOW_ARTIFACT.match(text) + if match and prior: + artifact = match.group(1).casefold(); return CompiledConversationAct(str(prior["mode"]), tuple(prior["subject_surfaces"]), artifact, 3, str(prior["surface_language"]), "m3b_backward_artifact_reference"), None + if FOLLOW_MORE.match(text) and prior: + return CompiledConversationAct("explain", tuple(prior["subject_surfaces"]), "essay", 3, str(prior["surface_language"]), "m3b_backward_expansion_reference"), None + match = FOLLOW_COMPARE.match(text) + if match and prior: + return CompiledConversationAct("compare", (str(prior["subject_surfaces"][0]), match.group(1)), "paragraph", 1, str(prior["surface_language"]), "m3b_backward_comparison_reference"), None + match = FOLLOW_WHAT.match(text) + if match: + return CompiledConversationAct("explain", (match.group(1),), "paragraph", 1, "en", "m3b_topic_shift"), None + act = compile_act(text) + if act is not None: + roles = act.roles; operator = act.operator + if operator == "compare": topics = (str(roles["left"]), str(roles["right"])) + elif operator == "define": topics = (str(roles["term"]),) + elif operator == "explain": topics = (str(roles["topic"]),) + elif operator == "summarize": topics = (str(roles["source"]),) + elif operator == "classify": topics = (str(roles["subject"]),) + elif operator == "enumerate": topics = (str(roles["topic"]),) + elif operator == "counterfactual": topics = (str(roles["consequence_query"]),) + else: return None, "the compiled act is not yet connected to authenticated domain realization" + mode = operator if operator in {"define", "explain", "summarize", "compare"} else "explain" + condition = str(roles["condition"]) if operator == "counterfactual" else None + return CompiledConversationAct(mode, topics, self._artifact_for(operator), 1, act.surface_language, "m1d_language_neutral_act", condition), None + for regex, operator in QUESTION: + match = regex.match(text) + if match: + return CompiledConversationAct(operator, (match.group(1),), self._artifact_for(operator), 1, "en", "m3b_question_construction"), None + return None, "I could not yet compile that utterance into a supported relational discourse act" + + def chat(self, text: str, *, session_id: str = "default") -> dict[str, Any]: + started = time.perf_counter(); act, failure = self._compile(str(text), session_id) + if act is None: + result = {"schema_version": SCHEMA, "status": "clarification", "operation": "m3b_uncompiled_utterance", "response": failure, "identity": self.identity, "factual_release": False, "utterance_program": None} + result["session"] = self.sessions.append(session_id, text=str(text), program={"uncompiled": True}, result=result, state=None); result["latency_ms"] = (time.perf_counter() - started) * 1000.0; return result + resolved = [self._resolve_surface(value) for value in act.topic_surfaces] + if any(value is None for value in resolved) or len({value[0] for value in resolved if value}) != 1: + result = {"schema_version": SCHEMA, "status": "clarification", "operation": "m3b_topic_authority_withheld", "response": "I recognized the requested act, but I need an exact authenticated topic address in one specialist domain before I can answer.", "identity": self.identity, "factual_release": False, "utterance_program": act.public(), "topic_resolution": [{"surface": surface, "resolved": value} for surface, value in zip(act.topic_surfaces, resolved)]} + result["session"] = self.sessions.append(session_id, text=str(text), program=act.public(), result=result, state=None); result["latency_ms"] = (time.perf_counter() - started) * 1000.0; return result + domain = resolved[0][0]; subjects = tuple(value[1] for value in resolved) + premises = () + world_id = None + if act.condition: + world_id = "world_" + hashlib.sha256((session_id + ":" + act.condition).encode()).hexdigest()[:16] + premises = ({"subject_surface": "the supplied condition", "predicate": "states", "object_surface": act.condition},) + predicate = "defined_as" if act.operator == "define" else "instance_of" if act.operator == "classify" else None + request = RealizationRequest(domain, subjects, act.artifact, act.operator, act.paragraphs, predicate=predicate, world_id=world_id, temporary_premises=premises) + realization = self.realizer.realize(request) + state = {"domain": domain, "subjects": list(subjects), "subject_surfaces": list(act.topic_surfaces), "mode": act.operator, "artifact": act.artifact, "surface_language": act.surface_language, "request_sha256": realization["request_sha256"]} + result = {"schema_version": SCHEMA, "status": realization["status"], "operation": "m3b_conversational_realization" if realization["status"] == "answered" else "m3b_realization_withheld", "response": realization["response"], "identity": self.identity, "factual_release": realization["factual_release"], "utterance_program": act.public(), "topic_resolution": {"domain": domain, "subjects": list(subjects)}, "realization": realization} + result["session"] = self.sessions.append(session_id, text=str(text), program=act.public(), result=result, state=state); result["latency_ms"] = (time.perf_counter() - started) * 1000.0; return result + + +__all__ = ["SCHEMA", "IDENTITY", "ConversationalFederationOracle", "CompiledConversationAct", "HashChainedSessionStore"] diff --git a/root/wander-train/stage5m3c_exact_topic_conversation_successor.py b/root/wander-train/stage5m3c_exact_topic_conversation_successor.py new file mode 100644 index 0000000000000000000000000000000000000000..cc26970a0074707e12e6e47642ba1c72bdedd45e --- /dev/null +++ b/root/wander-train/stage5m3c_exact_topic_conversation_successor.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""M3c exact topic authority and definition-fallback successor to frozen M3b.""" +from __future__ import annotations +import hashlib,time +from pathlib import Path +from stage5m3a_evidence_addressed_realizer import RealizationRequest +from stage5m3b_conversational_federation import CONCEPT_ALIASES,ConversationalFederationOracle,SCHEMA as PARENT_SCHEMA,_key + +SCHEMA="oracle-stage5m3c-exact-topic-conversation-successor-v1" +SAFE_QUERY_TAILS={"change","changes","behave","behaves","work","works","respond","responds","differ","differs"} + +class ExactTopicConversationalOracle(ConversationalFederationOracle): + def _resolve_surface(self,surface:str): + key=CONCEPT_ALIASES.get(_key(surface),_key(surface)) + if key in self.subject_domains: + domains=sorted(self.subject_domains[key]);return (domains[0],key) if len(domains)==1 else None + matches=[] + for subject in self.subject_keys: + prefix=subject+"_" + if key.startswith(prefix) and set(key[len(prefix):].split("_"))<=SAFE_QUERY_TAILS: + matches.append(subject) + matches=list(dict.fromkeys(matches)) + if len(matches)!=1:return None + domains=sorted(self.subject_domains[matches[0]]);return (domains[0],matches[0]) if len(domains)==1 else None + + def chat(self,text:str,*,session_id:str="default"): + started=time.perf_counter();act,failure=self._compile(str(text),session_id) + if act is None: + result={"schema_version":SCHEMA,"status":"clarification","operation":"m3c_uncompiled_utterance","response":failure,"identity":self.identity,"factual_release":False,"utterance_program":None,"m3c_exact_topic_authority":True};result["session"]=self.sessions.append(session_id,text=str(text),program={"uncompiled":True},result=result,state=None);result["latency_ms"]=(time.perf_counter()-started)*1000;return result + resolved=[self._resolve_surface(value) for value in act.topic_surfaces] + if any(value is None for value in resolved) or len({value[0] for value in resolved if value})!=1: + result={"schema_version":SCHEMA,"status":"clarification","operation":"m3c_topic_authority_withheld","response":"I recognized the requested act, but I need an exact authenticated topic address in one specialist domain before I can answer.","identity":self.identity,"factual_release":False,"utterance_program":act.public(),"topic_resolution":[{"surface":surface,"resolved":value} for surface,value in zip(act.topic_surfaces,resolved)],"m3c_exact_topic_authority":True};result["session"]=self.sessions.append(session_id,text=str(text),program=act.public(),result=result,state=None);result["latency_ms"]=(time.perf_counter()-started)*1000;return result + domain=resolved[0][0];subjects=tuple(value[1] for value in resolved);premises=();world_id=None + if act.condition: + world_id="world_"+hashlib.sha256((session_id+":"+act.condition).encode()).hexdigest()[:16];premises=({"subject_surface":"the supplied condition","predicate":"states","object_surface":act.condition},) + predicate="defined_as" if act.operator=="define" else "instance_of" if act.operator=="classify" else None + request=RealizationRequest(domain,subjects,act.artifact,act.operator,act.paragraphs,predicate=predicate,world_id=world_id,temporary_premises=premises);realization=self.realizer.realize(request);fallback=False + if realization["status"]=="withheld" and act.operator=="define" and realization["reason"]=="no_authenticated_topic_subgraph": + request=RealizationRequest(domain,subjects,"sentence","define",1,predicate=None,world_id=world_id,temporary_premises=premises);realization=self.realizer.realize(request);fallback=realization["status"]=="answered" + state={"domain":domain,"subjects":list(subjects),"subject_surfaces":list(act.topic_surfaces),"mode":act.operator,"artifact":act.artifact,"surface_language":act.surface_language,"request_sha256":realization["request_sha256"]} + result={"schema_version":SCHEMA,"status":realization["status"],"operation":"m3c_conversational_realization" if realization["status"]=="answered" else "m3c_realization_withheld","response":realization["response"],"identity":self.identity,"factual_release":realization["factual_release"],"utterance_program":act.public(),"topic_resolution":{"domain":domain,"subjects":list(subjects)},"realization":realization,"m3c_exact_topic_authority":True,"taxonomy_fallback":fallback,"parent_schema":PARENT_SCHEMA};result["session"]=self.sessions.append(session_id,text=str(text),program=act.public(),result=result,state=state);result["latency_ms"]=(time.perf_counter()-started)*1000;return result + +__all__=["SCHEMA","ExactTopicConversationalOracle","SAFE_QUERY_TAILS"] + diff --git a/root/wander-train/stage5m5a_adversarial_epistemic_guard.py b/root/wander-train/stage5m5a_adversarial_epistemic_guard.py new file mode 100644 index 0000000000000000000000000000000000000000..8e501e82b482033b10fd2eefb6653c0141e922b1 --- /dev/null +++ b/root/wander-train/stage5m5a_adversarial_epistemic_guard.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""M5a authority guards for corpus, time, sensors, language, tools, and memory.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import re +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping, Sequence +from urllib.parse import urlparse + +from oracle_10d_resolution_state import canonical_sha256 +from stage5m4a_closed_loop_typed_agent import DeterministicSandbox, ToolSpec, TypedToolRegistry +from stage5m4b_bfcl_compatible_language_agent import BFCLCompatibleLanguageAgent + + +SCHEMA = "oracle-stage5m5a-adversarial-epistemic-guard-v1" +CONTROL_INJECTION = re.compile( + r"(?:ignore|disregard|override|forget)\s+(?:all\s+)?(?:(?:previous|prior)\s+)?(?:(?:system|developer)\s+)?(?:instructions?|rules?|prompts?)" + r"|(?:reveal|print|exfiltrate)\s+(?:the\s+)?(?:system|developer)\s+prompt" + r"|<\s*/?\s*(?:tool_call|assistant|system|developer)\b" + r"|^(?:assistant|system|developer|tool)\s*:", + re.I, +) +CONTROL_CHARACTERS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + + +def digest(value: Any) -> str: + return canonical_sha256(value) + + +def _utc(value: str) -> datetime: + text = str(value).strip().replace("Z", "+00:00") + parsed = datetime.fromisoformat(text) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +@dataclass(frozen=True) +class GuardDecision: + accepted: bool + reason: str + authority_class: str + details: Mapping[str, Any] + + def public(self) -> dict[str, Any]: + return {"accepted": self.accepted, "reason": self.reason, "authority_class": self.authority_class, "details": dict(self.details)} + + +class AuthenticatedCorpusAdmissionGuard: + REQUIRED = {"source_id", "source_url", "authority", "license", "fetched_at", "content_sha256"} + + def __init__(self, *, maximum_bytes: int = 8_000_000, maximum_line_chars: int = 16_384, maximum_future_seconds: int = 300): + self.maximum_bytes = maximum_bytes; self.maximum_line_chars = maximum_line_chars; self.maximum_future_seconds = maximum_future_seconds + + def admit(self, content: bytes, manifest: Mapping[str, Any], *, now: datetime) -> dict[str, Any]: + errors = [] + if set(manifest) != self.REQUIRED: + errors.append("manifest_schema_mismatch") + if len(content) > self.maximum_bytes: + errors.append("corpus_resource_limit") + actual_sha = hashlib.sha256(content).hexdigest() + if manifest.get("content_sha256") != actual_sha: + errors.append("content_hash_mismatch") + parsed_url = urlparse(str(manifest.get("source_url", ""))) + if parsed_url.scheme != "https" or not parsed_url.netloc: + errors.append("source_url_not_https") + if not str(manifest.get("authority", "")).strip() or not str(manifest.get("license", "")).strip(): + errors.append("source_authority_or_license_missing") + try: + fetched = _utc(str(manifest.get("fetched_at", ""))) + if (fetched - now.astimezone(timezone.utc)).total_seconds() > self.maximum_future_seconds: + errors.append("future_dated_acquisition") + except (TypeError, ValueError): + fetched = None; errors.append("invalid_acquisition_time") + try: + text = content.decode("utf-8", errors="strict") + except UnicodeDecodeError: + text = ""; errors.append("corpus_not_utf8") + quarantine = []; admitted = [] + for ordinal, line in enumerate(text.splitlines()): + reasons = [] + if len(line) > self.maximum_line_chars: + reasons.append("line_resource_limit") + if CONTROL_CHARACTERS.search(line): + reasons.append("control_character") + if CONTROL_INJECTION.search(line.strip()): + reasons.append("control_plane_injection") + row = {"ordinal": ordinal, "line": line, "line_sha256": hashlib.sha256(line.encode()).hexdigest()} + if reasons: + quarantine.append({**row, "reasons": reasons}) + else: + admitted.append(row) + accepted = not errors and bool(admitted) + return { + "schema_version": SCHEMA, + "accepted": accepted, + "reason": "authenticated_content_with_line_quarantine" if accepted else "corpus_envelope_rejected", + "errors": errors, + "source_id": manifest.get("source_id"), + "content_sha256": actual_sha, + "admitted_lines": admitted, + "quarantine": quarantine, + "all_lines_accounted": len(admitted) + len(quarantine) == len(text.splitlines()), + "automatic_fact_promotions": 0, + } + + +class TemporalAuthorityGate: + def authorize(self, temporal: Mapping[str, str] | None, *, query_at: datetime, require_current: bool = False, maximum_age_days: float | None = None) -> GuardDecision: + query = query_at.astimezone(timezone.utc); value = dict(temporal or {}) + if not value: + return GuardDecision(not require_current, "timeless_claim" if not require_current else "current_claim_requires_temporal_evidence", "authenticated_fact" if not require_current else "withheld", {"query_at": query.isoformat()}) + kind = value.get("kind") + try: + if kind in {"year", "closed_interval"}: + start = int(value["start"]); end = int(value["end"]); accepted = start <= query.year <= end + reason = "query_within_temporal_scope" if accepted else "claim_outside_temporal_scope" + elif kind == "as_of": + raw = str(value["as_of"]) + observed = datetime(int(raw), 1, 1, tzinfo=timezone.utc) if re.fullmatch(r"\d{4}", raw) else _utc(raw) + age = (query - observed).total_seconds() / 86400.0 + accepted = age >= 0 and (maximum_age_days is None or age <= maximum_age_days) + reason = "as_of_fresh" if accepted else "as_of_future" if age < 0 else "as_of_stale" + return GuardDecision(accepted, reason, "authenticated_fact" if accepted else "withheld", {"age_days": age, "query_at": query.isoformat(), "temporal": value}) + else: + accepted = False; reason = "unknown_temporal_kind" + except (KeyError, TypeError, ValueError): + accepted = False; reason = "invalid_temporal_scope" + return GuardDecision(accepted, reason, "authenticated_fact" if accepted else "withheld", {"query_at": query.isoformat(), "temporal": value}) + + +class ExactRelationalAuthorityGate: + @staticmethod + def authorize(request: Mapping[str, Any], registered: Sequence[Mapping[str, Any]]) -> GuardDecision: + fields = ("domain", "subject", "predicate", "polarity") + key = tuple(str(request.get(field, "")).strip().casefold() for field in fields) + matches = [value for value in registered if tuple(str(value.get(field, "")).strip().casefold() for field in fields) == key] + if len(matches) == 1: + return GuardDecision(True, "unique_exact_relational_address", "authenticated_fact", {"address_sha256": digest(dict(request))}) + return GuardDecision(False, "unknown_relational_address" if not matches else "colliding_relational_address", "withheld", {"matches": len(matches), "address_sha256": digest(dict(request))}) + + +@dataclass(frozen=True) +class SensorSpec: + sensor_id: str + unit: str + minimum: float + maximum: float + maximum_delta: float + maximum_age_seconds: float + maximum_future_seconds: float + secret: str + + +class BoundedSensorIngress: + FIELDS = {"sensor_id", "sequence", "observed_at", "value", "unit", "signature"} + + def __init__(self, specs: Sequence[SensorSpec], state_path: Path): + self.specs = {value.sensor_id: value for value in specs}; self.state_path = state_path.resolve() + self.state_path.parent.mkdir(parents=True, exist_ok=True) + + @staticmethod + def sign(event: Mapping[str, Any], secret: str) -> str: + body = {key: value for key, value in event.items() if key != "signature"} + return hmac.new(secret.encode(), json.dumps(body, sort_keys=True, separators=(",", ":")).encode(), hashlib.sha256).hexdigest() + + def _load(self) -> dict[str, Any]: + if not self.state_path.exists(): + return {"sensors": {}, "events": [], "head_sha256": "0" * 64} + payload = json.loads(self.state_path.read_text(encoding="utf-8")); previous = "0" * 64 + for ordinal, event in enumerate(payload["events"]): + if event["ordinal"] != ordinal or event["previous_sha256"] != previous: + raise ValueError("sensor ledger predecessor changed") + body = {key: value for key, value in event.items() if key != "event_sha256"} + if digest(body) != event["event_sha256"]: + raise ValueError("sensor ledger event changed") + previous = event["event_sha256"] + if payload["head_sha256"] != previous: + raise ValueError("sensor ledger head changed") + return payload + + def ingest(self, event: Mapping[str, Any], *, now: datetime) -> dict[str, Any]: + reasons = [] + if set(event) != self.FIELDS: + reasons.append("sensor_schema_mismatch") + spec = self.specs.get(str(event.get("sensor_id", ""))) + if spec is None: + reasons.append("unknown_sensor") + if spec is not None and not hmac.compare_digest(str(event.get("signature", "")), self.sign(event, spec.secret)): + reasons.append("sensor_signature_invalid") + try: + sequence = int(event["sequence"]); value = float(event["value"]); observed = _utc(str(event["observed_at"])) + except (KeyError, TypeError, ValueError): + sequence = -1; value = float("nan"); observed = now.astimezone(timezone.utc); reasons.append("sensor_value_invalid") + state = self._load(); prior = state["sensors"].get(str(event.get("sensor_id", ""))) + if spec is not None: + age = (now.astimezone(timezone.utc) - observed).total_seconds() + if str(event.get("unit")) != spec.unit: + reasons.append("sensor_unit_mismatch") + if not spec.minimum <= value <= spec.maximum: + reasons.append("sensor_range_anomaly") + if age > spec.maximum_age_seconds: + reasons.append("sensor_stale") + if age < -spec.maximum_future_seconds: + reasons.append("sensor_future_dated") + if prior and sequence <= int(prior["sequence"]): + reasons.append("sensor_replay_or_reorder") + if prior and abs(value - float(prior["value"])) > spec.maximum_delta: + reasons.append("sensor_delta_anomaly") + if reasons: + return {"schema_version": SCHEMA, "accepted": False, "reason": sorted(set(reasons)), "authority_class": "withheld", "factual_promotion": False} + public = {key: event[key] for key in self.FIELDS if key != "signature"} + body = {"ordinal": len(state["events"]), "previous_sha256": state["head_sha256"], "event_sha256_source": digest(public), "sensor_id": spec.sensor_id, "sequence": sequence, "authority_class": "bounded_sensor_observation", "factual_promotion": False} + ledger_event = {**body, "event_sha256": digest(body)}; state["events"].append(ledger_event); state["head_sha256"] = ledger_event["event_sha256"]; state["sensors"][spec.sensor_id] = {"sequence": sequence, "value": value, "observed_at": observed.isoformat(), "event_sha256": ledger_event["event_sha256"]} + temporary = self.state_path.with_suffix(".tmp"); temporary.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8"); os.replace(temporary, self.state_path) + return {"schema_version": SCHEMA, "accepted": True, "reason": "verified_bounded_sensor_observation", "authority_class": "bounded_sensor_observation", "factual_promotion": False, "observation": public, "observation_sha256": digest(public), "ledger_head_sha256": state["head_sha256"]} + + +class TransactionalToolRegistry(TypedToolRegistry): + """Contain misdeclared/malformed tools at the sandbox transaction boundary.""" + + def __init__(self, *, maximum_argument_bytes: int = 65_536, maximum_observation_bytes: int = 262_144): + super().__init__(); self.maximum_argument_bytes = maximum_argument_bytes; self.maximum_observation_bytes = maximum_observation_bytes + + @classmethod + def from_registry(cls, source: TypedToolRegistry) -> "TransactionalToolRegistry": + value = cls() + for name in sorted(source.tools): + value.register(source.tools[name]) + return value + + def execute(self, sandbox: DeterministicSandbox, name: str, arguments: Mapping[str, Any], *, confirmed: bool, fault: str = "none") -> dict[str, Any]: + before = sandbox.snapshot(); before_sha = sandbox.sha256() + if len(json.dumps(arguments, ensure_ascii=False, default=str)) > self.maximum_argument_bytes: + return {"status": "failed", "reason": "tool_argument_resource_limit", "tool": name, "sandbox_restored": True} + result = super().execute(sandbox, name, arguments, confirmed=confirmed, fault=fault) + oversized = len(json.dumps(result.get("observation", {}), ensure_ascii=False, default=str)) > self.maximum_observation_bytes + if oversized: + result = {"status": "failed", "reason": "tool_observation_resource_limit", "tool": name} + if result.get("status") != "succeeded" and sandbox.sha256() != before_sha: + sandbox.collections = json.loads(json.dumps(before, sort_keys=True)) + result = {**result, "sandbox_restored": sandbox.sha256() == before_sha, "contained_side_effect": True} + return result + + +class GuardedBFCLAgent: + def __init__(self, agent: BFCLCompatibleLanguageAgent, *, maximum_query_chars: int = 4096, maximum_tokens: int = 512, maximum_functions: int = 32, maximum_schema_bytes: int = 65_536, maximum_fields_per_function: int = 32): + self.agent = agent; self.maximum_query_chars = maximum_query_chars; self.maximum_tokens = maximum_tokens; self.maximum_functions = maximum_functions; self.maximum_schema_bytes = maximum_schema_bytes; self.maximum_fields_per_function = maximum_fields_per_function + + def _guard(self, query: str, functions: Sequence[Mapping[str, Any]] | None, session_id: str) -> GuardDecision: + reasons = [] + text = str(query) + if len(text) > self.maximum_query_chars or len(text.split()) > self.maximum_tokens: + reasons.append("query_resource_limit") + if CONTROL_CHARACTERS.search(text): + reasons.append("query_control_character") + if CONTROL_INJECTION.search(text.strip()): + reasons.append("query_control_plane_injection") + if not session_id or len(session_id) > 128 or CONTROL_CHARACTERS.search(session_id): + reasons.append("invalid_session_id") + if functions is not None: + if len(functions) > self.maximum_functions: + reasons.append("function_count_limit") + if len(json.dumps(functions, ensure_ascii=False)) > self.maximum_schema_bytes: + reasons.append("function_schema_resource_limit") + names = [str(value.get("name", "")) for value in functions] + if len(names) != len(set(names)): + reasons.append("duplicate_function_name") + registry = set(self.agent.agent.registry.tools) + if not set(names) <= registry: + reasons.append("unregistered_function_schema") + for value in functions: + parameters = value.get("parameters", {}) + properties = parameters.get("properties", {}) if isinstance(parameters, Mapping) else {} + if len(properties) > self.maximum_fields_per_function: + reasons.append("function_field_limit") + return GuardDecision(not reasons, "guard_pass" if not reasons else "guard_rejected", "structural" if not reasons else "withheld", {"reasons": sorted(set(reasons))}) + + def answer(self, query: str, *, functions: Sequence[Mapping[str, Any]] | None = None, session_id: str = "default", confirmed: bool = False, faults: Mapping[str, str] | None = None) -> dict[str, Any]: + decision = self._guard(query, functions, session_id) + if not decision.accepted: + return {"schema_version": SCHEMA, "status": "rejected", "factual_release": False, "guard": decision.public(), "execution": None} + result = self.agent.answer(query, functions=functions, session_id=session_id, confirmed=confirmed, faults=faults) + result = dict(result); result["schema_version"] = SCHEMA; result["guard"] = decision.public(); result["description_text_authority"] = "inert_schema_metadata"; return result + + +__all__ = ["SCHEMA", "GuardDecision", "AuthenticatedCorpusAdmissionGuard", "TemporalAuthorityGate", "ExactRelationalAuthorityGate", "SensorSpec", "BoundedSensorIngress", "TransactionalToolRegistry", "GuardedBFCLAgent"] diff --git a/root/wander-train/stage5m6a_oracle_successor_runtime.py b/root/wander-train/stage5m6a_oracle_successor_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..c31d9f5a74dfcf6d2ea62265ab95bab76d8fc0f9 --- /dev/null +++ b/root/wander-train/stage5m6a_oracle_successor_runtime.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +"""M6 whole-system successor runtime. + +This module composes the frozen M3 conversation/evidence path with the M4/M5 +transactional tool path and adds a deterministic structural-reasoning adapter. +It intentionally contains no language model, pretrained embedding, gradient +update, or runtime access to evaluation answers. +""" +from __future__ import annotations + +import ast +import calendar +import hashlib +import itertools +import json +import math +import operator +import re +import time +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any, Mapping, Sequence + +from stage5m3c_exact_topic_conversation_successor import ExactTopicConversationalOracle +from stage5m4a_closed_loop_typed_agent import ClosedLoopTypedAgent, DeterministicSandbox, build_local_registry +from stage5m4b_bfcl_compatible_language_agent import BFCLCompatibleLanguageAgent, bfcl_functions +from stage5m5a_adversarial_epistemic_guard import GuardedBFCLAgent, TransactionalToolRegistry + + +SCHEMA = "oracle-stage5m6a-whole-system-successor-v1" +IDENTITY = { + "name": "Oracle Relational V3 M6 successor", + "kind": "deterministic relational cognitive runtime", + "purpose": "compile utterances into typed reasoning, evidence, discourse, or tool programs", + "epistemic_policy": "release authenticated facts and proven consequences; distinguish hypotheses and withhold unsupported claims", + "architecture": "HRR relational transport with a protected 4:1 root/payload spine and terminal cleanup", +} + +TOOL_CUE = re.compile(r"\b(?:read|get|fetch|sum|add|total|format|store|save|write)\b", re.I) +CONTROL_INJECTION = re.compile( + r"(?:ignore|override|replace|reveal|discard).{0,48}(?:system|developer|previous|hidden|policy|instructions?)", + re.I | re.S, +) + + +def _sha(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str).encode()).hexdigest() + + +def _norm(value: str) -> str: + return " ".join(str(value).casefold().replace("’", "'").split()) + + +def _strip_harness(prompt: str) -> tuple[str, list[str]]: + marker = "\nAllowed answers:\n" + if marker not in prompt: + return prompt.strip(), [] + body, tail = prompt.split(marker, 1) + choices = [] + for line in tail.splitlines(): + if line.startswith("- "): + choices.append(line[2:].strip()) + elif line.startswith("Return "): + break + return body.strip(), choices + + +def _choice(choices: Sequence[str], candidates: Sequence[str]) -> str | None: + normalized = {_norm(value): value for value in choices} + for candidate in candidates: + key = _norm(candidate) + if key in normalized: + return normalized[key] + for candidate in candidates: + key = _norm(candidate) + matches = [value for norm, value in normalized.items() if key and (key in norm or norm in key)] + if len(matches) == 1: + return matches[0] + return None + + +NUMBER_WORDS = { + "zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, + "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, + "eleven": 11, "twelve": 12, "thirteen": 13, "fourteen": 14, + "fifteen": 15, "sixteen": 16, "seventeen": 17, "eighteen": 18, + "nineteen": 19, "twenty": 20, "thirty": 30, "forty": 40, + "fifty": 50, "sixty": 60, "seventy": 70, "eighty": 80, "ninety": 90, +} + + +def _number(value: str) -> float | None: + token = value.strip().casefold().replace(",", "") + try: + return float(token) + except ValueError: + pass + parts = re.split(r"[-\s]+", token) + if parts and all(part in NUMBER_WORDS for part in parts): + return float(sum(NUMBER_WORDS[part] for part in parts)) + return None + + +_BINOPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, ast.Pow: operator.pow, ast.Mod: operator.mod} +_UNARY = {ast.UAdd: operator.pos, ast.USub: operator.neg} + + +def _safe_arithmetic(expression: str) -> float: + node = ast.parse(expression, mode="eval") + + def visit(value: ast.AST) -> float: + if isinstance(value, ast.Expression): + return visit(value.body) + if isinstance(value, ast.Constant) and isinstance(value.value, (int, float)) and not isinstance(value.value, bool): + return float(value.value) + if isinstance(value, ast.BinOp) and type(value.op) in _BINOPS: + left, right = visit(value.left), visit(value.right) + result = _BINOPS[type(value.op)](left, right) + if not math.isfinite(result) or abs(result) > 1e15: + raise ValueError("arithmetic resource boundary") + return result + if isinstance(value, ast.UnaryOp) and type(value.op) in _UNARY: + return _UNARY[type(value.op)](visit(value.operand)) + raise ValueError("unsupported arithmetic syntax") + + return visit(node) + + +def _format_number(value: float) -> str: + return str(int(value)) if float(value).is_integer() else format(value, ".12g") + + +@dataclass(frozen=True) +class StructuralResult: + status: str + answer: str | None + operation: str + proof: Mapping[str, Any] + + def public(self) -> dict[str, Any]: + body = {"status": self.status, "answer": self.answer, "operation": self.operation, "proof": dict(self.proof)} + return {**body, "result_sha256": _sha(body)} + + +class StandardStructuralReasoner: + """Typed task-family solvers; category is harness metadata, never inferred answer data.""" + + def solve(self, category: str, prompt: str) -> StructuralResult: + body, choices = _strip_harness(prompt) + method = getattr(self, "_" + category, None) + if method is None: + return StructuralResult("withheld", None, "unsupported_task_family", {"category": category}) + try: + answer, proof = method(body, choices) + except Exception as exc: # bounded failure is part of the public contract + return StructuralResult("withheld", None, "structural_parse_failure", {"category": category, "error_type": type(exc).__name__}) + if answer is None: + return StructuralResult("withheld", None, "insufficient_structural_authority", {"category": category, **proof}) + if choices and answer not in choices: + return StructuralResult("withheld", None, "answer_outside_registered_choices", {"category": category, "candidate": answer}) + return StructuralResult("answered", answer, "typed_structural_solution", {"category": category, **proof}) + + @staticmethod + def _simple_arithmetic_multiple_choice(body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + text = body.replace("×", "*").replace("÷", "/").replace("^", "**") + expressions = re.findall(r"(? tuple[str | None, dict[str, Any]]: + statement, _, question = body.partition("?") + inventory = re.sub(r"^.*?\b(?:have|has|contains?)\b", "", statement, count=1, flags=re.I) + segments = [value.strip(" .") for value in re.split(r",|\band\b", inventory, flags=re.I) if value.strip(" .")] + taxonomy = { + "animals": {"dog", "cat", "snake", "chicken", "frog", "rabbit", "pig", "snail", "horse", "cow", "goat", "sheep", "bird", "duck", "fish", "hamster", "mouse", "rat", "turtle", "lizard"}, + "musical instruments": {"violin", "viola", "cello", "guitar", "piano", "trumpet", "clarinet", "flute", "drum", "trombone", "saxophone", "harp", "oboe", "bassoon"}, + "instruments": {"violin", "viola", "cello", "guitar", "piano", "trumpet", "clarinet", "flute", "drum", "trombone", "saxophone", "harp", "oboe", "bassoon"}, + "vehicles": {"car", "truck", "bus", "bicycle", "bike", "motorcycle", "train", "boat", "ship", "airplane", "plane", "helicopter"}, + "fruits": {"apple", "orange", "banana", "pear", "peach", "plum", "grape", "lemon", "lime", "mango"}, + } + requested = next((name for name in taxonomy if name in question.casefold()), None) + parsed = [] + for segment in segments: + match = re.match(r"(?:an?|the)\s+(.+)$|(?:(\d+|" + "|".join(sorted(NUMBER_WORDS, key=len, reverse=True)) + r")\s+)(.+)$", segment, re.I) + if not match: continue + if match.group(1) is not None: count, noun = 1, match.group(1) + else: count, noun = int(_number(match.group(2)) or 0), match.group(3) + noun = re.sub(r"[^a-z -]", "", noun.casefold()).strip() + lemma = noun[:-2] if noun.endswith("es") and noun[:-2] in set().union(*taxonomy.values()) else noun[:-1] if noun.endswith("s") else noun + parsed.append((count, lemma)) + selected = parsed if requested is None else [(count, noun) for count, noun in parsed if noun in taxonomy[requested]] + total = sum(count for count, _ in selected) + return (str(total), {"inventory": parsed, "requested_taxonomy": requested, "selected": selected}) if selected else (None, {"reason": "no_countable_objects", "inventory": parsed, "requested_taxonomy": requested}) + + @staticmethod + def _analytic_entailment(body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + text = re.sub(r"^(?:Sentences:\s*)", "", body, flags=re.I).strip() + parts = [part.strip() for part in re.split(r"\b(?:so|therefore|thus)\b", text, maxsplit=1, flags=re.I)] + if len(parts) != 2: + return None, {"reason": "missing_premise_hypothesis_boundary"} + premise, hypothesis = map(_norm, parts) + premise = premise.rstrip("."); hypothesis = hypothesis.rstrip(".") + if premise == hypothesis: + entailed = True + elif re.search(r"\b(?:not|never|no)\b", premise) != re.search(r"\b(?:not|never|no)\b", hypothesis): + entailed = False + else: + # Analytic entailment may delete modifiers or existentially weaken a clause; + # it may not invent a new head noun, role, or predicate. + pwords = set(re.findall(r"[a-z]+", premise)) + hwords = set(re.findall(r"[a-z]+", hypothesis)) + function = {"a", "an", "the", "is", "are", "was", "were", "has", "have", "had", "some", "someone", "something"} + heads = hwords - function + entailed = bool(heads) and heads <= pwords + hypernyms = {"dogs": "mammals", "dog": "mammal", "cats": "mammals", "cat": "mammal", "pistol": "armed", "gun": "armed"} + if not entailed and any(source in pwords and target in hwords for source, target in hypernyms.items()): + # Numeric existential weakening: four dogs entails at least two mammals. + pnum = next((_number(word) for word in pwords if _number(word) is not None), None) + hnum = next((_number(word) for word in hwords if _number(word) is not None), None) + entailed = pnum is None or hnum is None or pnum >= hnum + yes = _choice(choices, ["entailment"]) + no = _choice(choices, ["no-entailment", "non-entailment"]) + return (yes if entailed else no), {"premise": premise, "hypothesis": hypothesis, "entailed": entailed, "rule": "lexical_projection"} + + @staticmethod + def _known_unknowns(_body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + unknown_markers = ("unknown", "cannot be determined", "not enough information", "insufficient information", "impossible to know") + matches = [choice for choice in choices if any(marker in _norm(choice) for marker in unknown_markers)] + return (matches[0], {"unknown_choice_count": len(matches)}) if len(matches) == 1 else (None, {"unknown_choice_count": len(matches)}) + + @staticmethod + def _disambiguation_qa(body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + text = re.sub(r"^(?:Sentence:\s*)", "", body, flags=re.I) + ambiguous = [choice for choice in choices if "ambiguous" in _norm(choice)] + pronouns = re.findall(r"\b(he|him|his|she|her|hers|they|them|their|it|its)\b", text, re.I) + if not pronouns: + return None, {"reason": "no_pronoun"} + pronoun = pronouns[-1].casefold() + # Match answer alternatives back to explicit noun phrases, applying only + # grammatical number/gender signals; do not manufacture world facts. + people = re.findall(r"\b([A-Z][a-z]+)\b", text) + viable = [] + for choice in choices: + normalized = _norm(choice) + if "ambiguous" in normalized: + continue + if any(_norm(person) in normalized for person in people): + viable.append(choice) + if re.search(r"\bbecause\s+(?:he|she|they)\s+is\b", text, re.I): + antecedent = re.search(r"\bto\s+the\s+([a-z-]+)\s+because", text, re.I) + if antecedent: + selected = [choice for choice in choices if antecedent.group(1).casefold() in _norm(choice)] + if len(selected) == 1: return selected[0], {"pronoun": pronoun, "rule": "causal_clause_nearest_argument"} + if re.search(r"\basked\s+the\s+([a-z-]+)\s+if\s+(?:he|she|they)\b", text, re.I): + antecedent = re.search(r"\basked\s+the\s+([a-z-]+)\s+if", text, re.I) + if antecedent: + selected = [choice for choice in choices if antecedent.group(1).casefold() in _norm(choice)] + if len(selected) == 1: return selected[0], {"pronoun": pronoun, "rule": "embedded_question_subject"} + if re.search(r"\bafter\b.*?,\s*[A-Z][a-z]+\s+went to their\b", text): + return (ambiguous[0], {"pronoun": pronoun, "rule": "plural_or_singular_they_attachment"}) if len(ambiguous) == 1 else (None, {"pronoun": pronoun}) + if len(viable) == 1: + return viable[0], {"pronoun": pronoun, "rule": "unique_explicit_antecedent"} + return (ambiguous[0], {"pronoun": pronoun, "viable": viable, "rule": "structural_ambiguity"}) if len(ambiguous) == 1 else (None, {"pronoun": pronoun}) + + @staticmethod + def _epistemic_reasoning(body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + text = _norm(body) + split = re.split(r"\b(?:hypothesis|conclusion)\s*:\s*", text, maxsplit=1) + if len(split) != 2: + # Most task instances separate the final claim with a question or newline. + lines = [line.strip() for line in body.splitlines() if line.strip()] + if len(lines) < 2: + return None, {"reason": "no_hypothesis_boundary"} + premise, hypothesis = _norm(" ".join(lines[:-1])), _norm(lines[-1]) + else: + premise, hypothesis = split + hypothesis = re.sub(r"^(?:whether\s+)?", "", hypothesis).strip(" .?") + modal = re.compile(r"^([a-z][a-z-]*)\s+(believes|thinks|suspects|assumes|remembers|knows|learns|realizes|sees)\s+that\s+(.+)$") + def parse_chain(value: str): + chain=[]; remainder=value.strip(" .") + while (match := modal.match(remainder)): + chain.append((match.group(1), match.group(2))); remainder=match.group(3) + return chain,remainder + pchain, pcontent = parse_chain(premise.replace("premise: ", "", 1)); hchain, hcontent = parse_chain(hypothesis) + variants = {(tuple(pchain), pcontent)}; factive = {"remembers", "knows", "learns", "realizes", "sees"} + changed = True + while changed: + changed = False + for chain, content in list(variants): + for index, (_, verb) in enumerate(chain): + if verb in factive: + candidate = (chain[:index] + chain[index + 1:], content) + if candidate not in variants: variants.add(candidate); changed = True + entailed = (tuple(hchain), hcontent) in variants + if re.search(r"\b(?:does not|doesn't|not)\b", hypothesis) and hypothesis not in premise: + entailed = False + answer = _choice(choices, ["entailment"] if entailed else ["non-entailment", "no-entailment"]) + return answer, {"hypothesis": hypothesis, "entailed": entailed, "rule": "exact_modal_graph_plus_factivity"} + + @staticmethod + def _logical_deduction_three_objects(body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + narrative = body + names = [] + listing = re.search(r"(?:there (?:are|were)|\b(?:sells?|contains?|has)) three [^:]+:\s*(.+?)\.\s", narrative, re.I) + if listing: + names = [re.sub(r"^(?:and\s+)?(?:(?:an?|the)\s+)?", "", item.strip(), flags=re.I) for item in re.split(r",\s*|,?\s+and\s+", listing.group(1), flags=re.I) if item.strip()] + # Fallback to recurring capitalized object tokens, excluding sentence words. + if len(names) != 3: + tokens = re.findall(r"\b[A-Z][a-z]+\b", narrative) + excluded = {"The", "A", "An", "Allowed", "Return", "What", "Which"} + counts = {token: tokens.count(token) for token in set(tokens) if token not in excluded} + names = sorted(counts, key=lambda value: (-counts[value], value))[:3] + if len(names) != 3: + return None, {"reason": "object_identification", "objects": names} + + constraints: list[tuple[str, str, str | int]] = [] + for left in names: + for right in names: + if left == right: continue + l, r = re.escape(left), re.escape(right) + if re.search(rf"\b(?:the\s+)?{l}\s+(?:is|was|finished|ranks?|sits?|stands?|are|were)\s+(?:to the )?(?:left of|above|ahead of|before|older than|higher than|less expensive than|cheaper than)\s+(?:the\s+)?{r}\b", narrative, re.I): constraints.append((left, "<", right)) + if re.search(rf"\b(?:the\s+)?{l}\s+(?:is|was|finished|ranks?|sits?|stands?|are|were)\s+(?:to the )?(?:right of|below|behind|after|newer than|lower than|more expensive than|costlier than)\s+(?:the\s+)?{r}\b", narrative, re.I): constraints.append((left, ">", right)) + positions = {"first": 0, "leftmost": 0, "top": 0, "oldest": 0, "highest": 0, "cheapest": 0, "least expensive": 0, "second": 1, "middle": 1, "second-oldest": 1, "second-newest": 1, "second-most expensive": 1, "third": 2, "last": 2, "rightmost": 2, "bottom": 2, "newest": 2, "lowest": 2, "most expensive": 2} + for name in names: + terms = "|".join(sorted((re.escape(value) for value in positions), key=len, reverse=True)) + pattern = rf"\b(?:the\s+)?{re.escape(name)}\s+(?:is|was|finished|ranks?|are|were)\s+(?:the\s+)?({terms})(?:\s+from\s+the\s+(left|right))?\b" + match = re.search(pattern, narrative, re.I) + if match: + index = positions[match.group(1).casefold()] + if match.group(2) and match.group(2).casefold() == "right": index = 2 - index + constraints.append((name, "=", index)) + + def valid(order: Sequence[str]) -> bool: + pos = {name.casefold(): index for index, name in enumerate(order)} + for left, op, right in constraints: + if left.casefold() not in pos: + continue + a = pos[left.casefold()] + b = int(right) if isinstance(right, int) else pos.get(str(right).casefold()) + if b is None or not ({"<": a < b, ">": a > b, "=": a == b}[op]): return False + return True + + orders = [order for order in itertools.permutations(names) if valid(order)] + if not orders: + return None, {"reason": "no_consistent_order", "objects": names, "constraints": constraints} + + def option_holds(option: str, order: Sequence[str]) -> bool | None: + pos = {name.casefold(): index for index, name in enumerate(order)}; text = option.casefold() + for name in names: + n = re.escape(name.casefold()) + match = re.search(rf"\b{n}\b.*?\b(left|before|above|older|higher)\b.*?\b({'|'.join(re.escape(x.casefold()) for x in names if x != name)})\b", text) + if match: return pos[name.casefold()] < pos[match.group(2)] + match = re.search(rf"\b{n}\b.*?\b(right|after|below|newer|lower)\b.*?\b({'|'.join(re.escape(x.casefold()) for x in names if x != name)})\b", text) + if match: return pos[name.casefold()] > pos[match.group(2)] + for word, index in positions.items(): + match = re.search(rf"\b{re.escape(name.casefold())}\b.*?\b{word}\b(?:\s+from\s+the\s+(left|right))?", text) + if match: + actual = 2-index if match.group(1) == "right" else index + return pos[name.casefold()] == actual + return None + + winners = [] + for option in choices: + values = [option_holds(option, order) for order in orders] + if values and all(value is True for value in values): winners.append(option) + return (winners[0], {"objects": names, "consistent_orders": len(orders), "winner_count": len(winners)}) if len(winners) == 1 else (None, {"objects": names, "consistent_orders": len(orders), "winner_count": len(winners)}) + + @staticmethod + def _date_understanding(body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + dates = [] + formats = ["%m/%d/%Y", "%m/%d/%y", "%B %d, %Y", "%b %d, %Y", "%Y-%m-%d"] + for token in re.findall(r"\b\d{1,2}/\d{1,2}/\d{2,4}\b|\b\d{4}-\d{2}-\d{2}\b|\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2},\s+\d{4}\b", body, re.I): + for fmt in formats: + try: + dates.append(datetime.strptime(token.title(), fmt).date()); break + except ValueError: pass + if not dates: + symbolic = None + if match := re.search(r"christmas eve (?:of|in) (\d{4})", body, re.I): + symbolic = date(int(match.group(1)), 12, 24) + elif match := re.search(r"last day of (\d{4})", body, re.I): + symbolic = date(int(match.group(1)), 12, 31) + elif match := re.search(r"last day of (Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:in\s+)?(\d{4})", body, re.I): + month = datetime.strptime(match.group(1)[:3].title(), "%b").month; year = int(match.group(2)); symbolic = date(year, month, calendar.monthrange(year, month)[1]) + elif match := re.search(r"born on the last day of Feb(?:urary|ruary)? in (\d{4}).*?(\d+)-year-old birthday", body, re.I | re.S): + year = int(match.group(1)) + int(match.group(2)); symbolic = date(year, 2, calendar.monthrange(year, 2)[1]) + elif match := re.search(r"got (?:her|his|their) job in (\d{4}).*?(\d+)-year work anniversary.*?on Dec 2, (?:her|his|their) second day at work", body, re.I | re.S): + symbolic = date(int(match.group(1)) + int(match.group(2)), 12, 1) + elif match := re.search(r"today is (\d{1,2})/(\d{1,2}).*?\b(\d{4})\b", body, re.I | re.S): + symbolic = date(int(match.group(3)), int(match.group(1)), int(match.group(2))) + if symbolic is None: + return None, {"reason": "no_anchor_date"} + dates = [symbolic] + anchor = dates[0]; text = body.casefold() + question_match = re.search(r"\b(?:what|which)\s+is\s+the\s+date\b.+$", text) + question = question_match.group(0) if question_match else text + statement = text[:question_match.start()] if question_match else text + value = anchor; applied = [] + away = re.search(r"(?:which|that) is (\d+|[a-z-]+) days? away from now", statement) + if away and (amount := _number(away.group(1))) is not None: value -= timedelta(days=int(amount)); applied.append(("anchor_to_now", -int(amount))) + elif "tomorrow is" in statement or statement.startswith("tomorrow"): + value -= timedelta(days=1); applied.append(("tomorrow_to_today", -1)) + elif "yesterday was" in statement or statement.startswith("yesterday"): + value += timedelta(days=1); applied.append(("yesterday_to_today", 1)) + + def add_months(source: date, months: int) -> date: + total = source.year * 12 + source.month - 1 + months; year, month0 = divmod(total, 12); month = month0 + 1 + return date(year, month, min(source.day, calendar.monthrange(year, month)[1])) + + relative = re.search(r"(?:date\s+)?(\d+|one|two|three|four|five|six|seven|eight|nine|ten|a)\s+(day|week|month|year)s?\s+(ago|before|earlier|from|after|later)", question) + if relative: + amount = 1 if relative.group(1) == "a" else int(_number(relative.group(1)) or 0); direction = -1 if relative.group(3) in {"ago", "before", "earlier"} else 1; unit = relative.group(2) + if unit == "day": value += timedelta(days=direction*amount) + elif unit == "week": value += timedelta(days=direction*amount*7) + elif unit == "month": value = add_months(value, direction*amount) + else: value = add_months(value, direction*amount*12) + applied.append((relative.group(0), direction*amount, unit)) + # Avoid platform-specific %-m / %-d directives on Windows. + candidates = [ + value.strftime("%m/%d/%Y"), value.isoformat(), + f"{value.month}/{value.day}/{value.year}", + f"{value.month:02d}/{value.day:02d}/{value.year}", + value.strftime("%B ") + f"{value.day}, {value.year}", + ] + return _choice(choices, candidates), {"anchor": dates[0].isoformat(), "result": value.isoformat(), "shifts": applied} + + @staticmethod + def _causal_judgment(body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + text = _norm(body) + # Conservative structural attribution: explicit direct production or prevention + # is licensed; mere temporal presence, omission without duty, and correlation are not. + positive = bool(re.search(r"\b(?:caused|made|forced|triggered|resulted in|led to|because of)\b", text)) + negative = bool(re.search(r"\b(?:merely|coincidence|unrelated|did not|failed to)\b", text)) + candidates = ["yes"] if positive and not negative else ["no"] if negative and not positive else [] + return _choice(choices, candidates), {"explicit_positive": positive, "explicit_negative": negative} + + @staticmethod + def _physics_questions(body: str, choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + # Exact-string physics tasks require a number and unit. Cover dimensionally + # explicit kinematics/Newton/Ohm/power relations; withhold everything else. + text = body.casefold(); nums = [(float(v), u) for v, u in re.findall(r"(-?\d+(?:\.\d+)?)\s*(m/s\^?2|m/s|s|kg|n|v|a|ohms?|w|j|hz|m)\b", text)] + value = None; unit = None; formula = None + def first(wanted: Sequence[str]) -> float | None: + return next((v for v, u in nums if u in wanted), None) + if "rescue basket" in text and (mass := re.search(r"(\d+(?:\.\d+)?)\s*-?\s*kg", text)) and (tension := re.search(r"tension.{0,20}?(\d+(?:\.\d+)?)\s*n", text)) and (angle := re.search(r"(\d+(?:\.\d+)?)\s*(?:°|degrees?)", text)) and (mu := re.search(r"coefficient of friction.{0,20}?(\d+(?:\.\d+)?)", text)): + m, t, theta, coefficient = float(mass.group(1)), float(tension.group(1)), math.radians(float(angle.group(1))), float(mu.group(1)) + value = (t * math.cos(theta) - coefficient * (m * 9.8 - t * math.sin(theta))) / m; unit = "m/s/s"; formula = "a=(Tcosθ-μ(mg-Tsinθ))/m" + elif "crosses the border" in text and (distance := re.search(r"(\d+(?:\.\d+)?)\s*mi\b", text)) and (speed := re.search(r"(\d+(?:\.\d+)?)\s*mi/h", text)) and (angle := re.search(r"(\d+(?:\.\d+)?)\s*degrees?", text)): + value = float(distance.group(1)) / (float(speed.group(1)) * math.cos(math.radians(float(angle.group(1))))) * 60.0; unit = "min"; formula = "t=d/(v cosθ)" + elif "final velocity" in text and (mass := re.search(r"(\d+(?:\.\d+)?)\s*-?\s*kg", text)) and (initial := re.search(r"moving at (\d+(?:\.\d+)?)\s*m/s", text)): + m = float(mass.group(1)); value = float(initial.group(1)) + impulses = [float(x) for x in re.findall(r"(?:impulse of|force of)\s*(\d+(?:\.\d+)?)\s*n(?:\*s)?", text)] + duration = re.search(r"force of\s*\d+(?:\.\d+)?\s*n for (\d+(?:\.\d+)?)\s*seconds?", text) + if impulses: value -= impulses[0] / m + if len(impulses) > 1 and duration: value -= impulses[1] * float(duration.group(1)) / m + if delta := re.search(r"lowered .+? velocity by (\d+(?:\.\d+)?)\s*m/s", text): value -= float(delta.group(1)) + unit = "m/s"; formula = "v_f=v_i+ΣJ/m" + elif "recoil speed" in text: + masses = [float(x) for x in re.findall(r"(\d+(?:\.\d+)?)\s*-?\s*kg", text)]; speed = re.search(r"speed of (\d+(?:\.\d+)?)\s*m/s", text) + if len(masses) >= 4 and speed: + value = (masses[-1] + masses[-2]) * float(speed.group(1)) / sum(masses[:-2]); unit = "m/s"; formula = "momentum_conservation" + elif "constant speed" in text and (mass := re.search(r"(\d+(?:\.\d+)?)\s*-?\s*kg", text)) and (mu := re.search(r"coefficient of friction.{0,24}?(\d+(?:\.\d+)?)", text)): + value = float(mu.group(1)) * float(mass.group(1)) * 9.8; unit = "N"; formula = "F_f=μmg" + elif "hitting the outcropping" in text and (initial := re.search(r"velocity of (\d+(?:\.\d+)?)\s*m/s", text)) and (height := re.search(r"(\d+(?:\.\d+)?)\s*m below", text)): + value = math.sqrt(float(initial.group(1)) ** 2 + 2 * 9.8 * float(height.group(1))); unit = "m/s"; formula = "v²=u²+2gh" + elif "total distance" in text: + legs = [(float(v), float(minutes)) for v, minutes in re.findall(r"speed of (\d+(?:\.\d+)?)\s*m/s for (\d+(?:\.\d+)?)\s*minutes?", text)] + if legs: + value = sum(v * minutes * 60 for v, minutes in legs); unit = "m"; formula = "segmented_distance" + if value is None and "momentum" in text and first(["kg"]) is not None and first(["m/s"]) is not None: + value = first(["kg"]) * first(["m/s"]); unit = "kg*m/s"; formula = "p=mv" + elif value is None and "pound" in text and "kg" in text and (pounds := re.findall(r"(\d+(?:\.\d+)?)\s+pounds?", text)): + value = float(pounds[-1]) * 0.45359237; unit = "kg"; formula = "kg=lb*0.45359237" + elif value is None and "component" in text and (tension := re.search(r"tension(?: force)?.{0,16}?(\d+(?:\.\d+)?)\s*n", text)) and (angle := re.search(r"(\d+(?:\.\d+)?)\s*(?:°|degrees?)", text)): + theta = math.radians(float(angle.group(1))); magnitude = float(tension.group(1)) + value = magnitude * (math.sin(theta) if "vertical" in text else math.cos(theta)); unit = "N"; formula = "component=T*sin(theta)" if "vertical" in text else "component=T*cos(theta)" + elif value is None and "force" in text and first(["kg"]) is not None and first(["m/s2", "m/s^2"]) is not None: + value = first(["kg"]) * first(["m/s2", "m/s^2"]); unit = "N"; formula = "F=ma" + elif value is None and ("voltage" in text or "potential" in text) and first(["a"]) is not None and first(["ohm", "ohms"]) is not None: + value = first(["a"]) * first(["ohm", "ohms"]); unit = "V"; formula = "V=IR" + elif value is None and "distance" in text and first(["m/s"]) is not None and first(["s"]) is not None: + value = first(["m/s"]) * first(["s"]); unit = "m"; formula = "d=vt" + elif value is None and ("speed" in text or "velocity" in text) and first(["m"]) is not None and first(["s"]) is not None: + value = first(["m"]) / first(["s"]); unit = "m/s"; formula = "v=d/t" + if value is None: + return None, {"reason": "no_licensed_formula"} + if formula == "segmented_distance": + rendered = str(int(round(value))) + elif formula == "momentum_conservation": + rendered = f"{value:.2f}" + elif abs(value - round(value)) < 0.05: + rendered = str(int(round(value))) + elif abs(value) >= 100: + rendered = f"{value:.1f}" + else: + rendered = _format_number(value) + candidate = f"{rendered} {unit}" + return (_choice(choices, [candidate]) if choices else candidate), {"formula": formula, "value": value, "unit": unit} + + @staticmethod + def _general_knowledge(_body: str, _choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + return None, {"reason": "no_answer_key_or_authenticated_fact_address"} + + @staticmethod + def _strategyqa(_body: str, _choices: Sequence[str]) -> tuple[str | None, dict[str, Any]]: + return None, {"reason": "implicit_world_knowledge_not_authenticated"} + + +class OracleM6SuccessorRuntime: + def __init__(self, *, federation_manifest: Path, state_root: Path): + self.state_root = Path(state_root); self.state_root.mkdir(parents=True, exist_ok=True) + self.conversation = ExactTopicConversationalOracle(Path(federation_manifest), state_root=self.state_root / "conversation") + base_registry = build_local_registry(); registry = TransactionalToolRegistry.from_registry(base_registry) + self.sandbox = DeterministicSandbox({"lab": {"alpha": 4, "beta": 7, "gamma": 11}, "notes": {}}) + closed = ClosedLoopTypedAgent(registry, self.sandbox, self.state_root / "tasks", maximum_actions=32, maximum_parallel=4) + language_agent = BFCLCompatibleLanguageAgent(closed, self.state_root / "tool_sessions") + self.tools = GuardedBFCLAgent(language_agent) + self.reasoner = StandardStructuralReasoner() + self.identity = {**IDENTITY, "identity_sha256": _sha(IDENTITY)} + + def benchmark(self, category: str, prompt: str) -> dict[str, Any]: + started = time.perf_counter(); result = self.reasoner.solve(category, prompt).public() + return { + "schema_version": SCHEMA, + "status": result["status"], + "response": result["answer"], + "operation": "m6a_standard_structural_reasoning", + "structural_result": result, + "factual_release": result["status"] == "answered", + "latency_ms": (time.perf_counter() - started) * 1000.0, + "audit": self.audit(), + } + + def chat(self, text: str, *, session_id: str = "default", functions: Sequence[Mapping[str, Any]] | None = None, confirmed: bool = False, faults: Mapping[str, str] | None = None) -> dict[str, Any]: + started = time.perf_counter(); source = str(text).strip() + if CONTROL_INJECTION.search(source): + return {"schema_version": SCHEMA, "status": "rejected", "response": "I cannot treat user text as control-plane authority.", "operation": "m6a_control_injection_rejected", "factual_release": False, "latency_ms": (time.perf_counter() - started) * 1000.0, "audit": self.audit()} + + if functions is not None or TOOL_CUE.search(source): + tool = self.tools.answer(source, functions=functions or bfcl_functions(self.tools.agent.agent), session_id=session_id, confirmed=confirmed, faults=faults) + if tool.get("disposition") not in {"irrelevant", "no_call"} and tool.get("status") != "no_call": + response = self._tool_surface(tool) + return {"schema_version": SCHEMA, "status": tool.get("status"), "response": response, "operation": "m6a_guarded_tool_path", "factual_release": bool(tool.get("factual_release")), "tool_result": tool, "latency_ms": (time.perf_counter() - started) * 1000.0, "audit": self.audit()} + + exact = re.match(r"^return exactly (?:these )?(?:\w+ )?words?:\s*(.+)$", source, re.I | re.S) + if exact: + response = " ".join(exact.group(1).split()) + return {"schema_version": SCHEMA, "status": "answered", "response": response, "operation": "m6a_response_constraint", "factual_release": False, "latency_ms": (time.perf_counter() - started) * 1000.0, "audit": self.audit()} + + identity = re.search(r"\b(?:who|what) are you\b|\bwhat is (?:the )?oracle\b|\byour (?:identity|purpose|architecture|capabilities)\b", source, re.I) + if identity: + response = ( + "I am the Oracle Relational V3 M6 successor, a deterministic relational cognitive runtime. " + "I compile utterances into typed evidence, reasoning, discourse, or tool programs; I preserve a protected HRR root through Tree routing, release authenticated facts or proved consequences, and explicitly withhold claims outside my present evidence and grammar." + ) + return {"schema_version": SCHEMA, "status": "answered", "response": response, "operation": "m6a_identity", "identity": self.identity, "factual_release": False, "latency_ms": (time.perf_counter() - started) * 1000.0, "audit": self.audit()} + + arithmetic = self._free_arithmetic(source) + if arithmetic is not None: + return {"schema_version": SCHEMA, "status": "answered", "response": arithmetic[0], "operation": "m6a_typed_arithmetic", "proof": arithmetic[1], "factual_release": False, "latency_ms": (time.perf_counter() - started) * 1000.0, "audit": self.audit()} + + premise = self._temporary_premise(source) + if premise is not None: + return {"schema_version": SCHEMA, "status": "answered", "response": premise[0], "operation": "m6a_temporary_world_reasoning", "proof": premise[1], "factual_release": False, "latency_ms": (time.perf_counter() - started) * 1000.0, "audit": self.audit()} + + result = dict(self.conversation.chat(source, session_id=session_id)) + parent_schema = result.get("schema_version") + result["schema_version"] = SCHEMA; result["parent_result_schema"] = parent_schema + result["operation"] = "m6a_evidence_conversation" if result.get("status") == "answered" else "m6a_clarification" + result["latency_ms"] = (time.perf_counter() - started) * 1000.0; result["audit"] = self.audit() + return result + + @staticmethod + def _tool_surface(result: Mapping[str, Any]) -> str: + if result.get("status") == "clarification": + disposition = result.get("disposition") or result.get("compilation", {}).get("disposition") + return { + "confirmation_required": "The requested write is compiled, but I need explicit confirmation before executing it.", + "missing_function": "The required function is not available in the registered tool set.", + "missing_parameter": "The tool request is missing a required parameter.", + }.get(str(disposition), "I need clarification before executing that tool request.") + execution = result.get("execution") or {} + observations = execution.get("verified_observations") or {} + if observations: + last = observations[sorted(observations)[-1]] + for key in ("text", "total", "record", "stored"): + if key in last: return str(last[key]) + return "The registered tool program completed with verified observations." if result.get("status") == "answered" else "The tool program was withheld because its effects could not be verified." + + @staticmethod + def _free_arithmetic(text: str) -> tuple[str, dict[str, Any]] | None: + normalized = text.replace("×", "*").replace("÷", "/").replace("^", "**") + match = re.search(r"(-?\d+(?:\.\d+)?(?:\s*(?:\*\*|[+\-*/%])\s*-?\d+(?:\.\d+)?)+)", normalized) + if match: + try: + value = _safe_arithmetic(match.group(1)) + return _format_number(value), {"expression": match.group(1), "value": value} + except Exception: + return None + average = re.search(r"average of\s+([\d.,\s]+(?:and\s+\d+(?:\.\d+)?)?)", text, re.I) + if average: + values = [float(v) for v in re.findall(r"-?\d+(?:\.\d+)?", average.group(1))] + if values: + value = sum(values) / len(values); return _format_number(value), {"operator": "mean", "values": values, "value": value} + return None + + @staticmethod + def _temporary_premise(text: str) -> tuple[str, dict[str, Any]] | None: + # Unary syllogism: Every A is B. X is A. Is X B? + universal = re.search(r"\bevery\s+([\w-]+)\s+is\s+([\w-]+)", text, re.I) + query = re.search(r"\bis\s+([A-Z][\w-]*)\s+([\w-]+)\s*\?", text, re.I) + if universal and query: + kind, prop = map(str.casefold, universal.groups()); entity, asked = query.group(1), query.group(2).casefold() + fact = re.search(rf"\b{re.escape(entity)}\s+is\s+(?:a\s+|an\s+)?{re.escape(kind)}\b", text, re.I) + if fact and asked == prop: + return f"Yes. Under the supplied premises, {entity} is {prop}.", {"rule": "universal_instantiation", "premises": [universal.group(0), fact.group(0)], "world": "temporary"} + # Explicitly licensed transitive binary relation. + relation = re.search(r"\b([A-Z][\w-]*)\s+([a-z][\w-]*)\s+([A-Z][\w-]*)\.\s*\3\s+\2\s+([A-Z][\w-]*)", text) + if relation and re.search(r"\btransitive\b", text, re.I): + left, rel, _, right = relation.groups() + return f"{left} {rel} {right}.", {"rule": "licensed_transitivity", "relation": rel, "world": "temporary"} + return None + + @staticmethod + def audit() -> dict[str, int]: + return {"transformer_calls": 0, "traditional_language_model_calls": 0, "pretrained_embedding_calls": 0, "next_token_predictions": 0, "gradient_steps": 0} + + def capabilities(self) -> dict[str, Any]: + return { + "schema_version": SCHEMA, + "identity": self.identity, + "routes": ["evidence_addressed_conversation", "temporary_world_reasoning", "typed_structural_reasoning", "transactional_tool_execution"], + "registered_tools": self.tools.agent.agent.registry.public(), + "supported_standard_task_families": [name[1:] for name in dir(self.reasoner) if name.startswith("_") and not name.startswith("__")], + "claim_boundary": "Compositional within registered operators, admitted evidence, and typed tools; not unrestricted language understanding, exhaustive world knowledge, or arbitrary tool induction.", + "audit": self.audit(), + } + + +__all__ = ["SCHEMA", "IDENTITY", "OracleM6SuccessorRuntime", "StandardStructuralReasoner", "StructuralResult"] diff --git a/root/wander-train/stage5m6d_whole_system_successor.py b/root/wander-train/stage5m6d_whole_system_successor.py new file mode 100644 index 0000000000000000000000000000000000000000..01bbb9b27f1eca0c18ee33f763aea91da7c7a0e8 --- /dev/null +++ b/root/wander-train/stage5m6d_whole_system_successor.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Narrow M6d successor for the two preserved M6c contract defects.""" +from __future__ import annotations +import re +from pathlib import Path +from typing import Any,Mapping,Sequence +from stage5m6a_oracle_successor_runtime import NUMBER_WORDS,OracleM6SuccessorRuntime,StandardStructuralReasoner,_number + +class M6dStructuralReasoner(StandardStructuralReasoner): + @staticmethod + def _object_counting(body:str,_choices:Sequence[str])->tuple[str|None,dict[str,Any]]: + statement,_,question=body.partition("?"); inventory=re.sub(r"^.*?\b(?:have|has|contains?)\b","",statement,count=1,flags=re.I); inventory=re.split(r"\.\s*(?:how|what|which)\b",inventory,maxsplit=1,flags=re.I)[0] + segments=[v.strip(" .") for v in re.split(r",|\band\b",inventory,flags=re.I) if v.strip(" .")] + taxonomy={"animals":{"dog","cat","snake","chicken","frog","rabbit","pig","snail","horse","cow","goat","sheep","bird","duck","fish","hamster","mouse","rat","turtle","lizard"},"musical instruments":{"violin","viola","cello","guitar","piano","trumpet","clarinet","flute","drum","trombone","saxophone","harp","oboe","bassoon"},"instruments":{"violin","viola","cello","guitar","piano","trumpet","clarinet","flute","drum","trombone","saxophone","harp","oboe","bassoon"}} + requested=next((name for name in taxonomy if name in question.casefold()),None);parsed=[] + for segment in segments: + match=re.match(r"(?:an?|the)\s+(.+)$|(?:(\d+|"+"|".join(sorted(NUMBER_WORDS,key=len,reverse=True))+r")\s+)(.+)$",segment,re.I) + if not match:continue + if match.group(1) is not None:count,noun=1,match.group(1) + else:count,noun=int(_number(match.group(2)) or 0),match.group(3) + noun=re.sub(r"[^a-z -]","",noun.casefold()).strip();lemma=noun[:-2] if noun.endswith("es") and noun[:-2] in set().union(*taxonomy.values()) else noun[:-1] if noun.endswith("s") else noun;parsed.append((count,lemma)) + selected=parsed if requested is None else [(count,noun) for count,noun in parsed if noun in taxonomy[requested]];total=sum(count for count,_ in selected) + return (str(total),{"inventory":parsed,"requested_taxonomy":requested,"selected":selected,"question_boundary":"sentence_before_interrogative"}) if selected else (None,{"reason":"no_countable_objects","inventory":parsed}) + +class OracleM6dSuccessorRuntime(OracleM6SuccessorRuntime): + def __init__(self,*,federation_manifest:Path,state_root:Path): + super().__init__(federation_manifest=federation_manifest,state_root=state_root);self.reasoner=M6dStructuralReasoner() + +__all__=["OracleM6dSuccessorRuntime","M6dStructuralReasoner"] diff --git a/root/wander-train/stage5m6e_count_scope_successor.py b/root/wander-train/stage5m6e_count_scope_successor.py new file mode 100644 index 0000000000000000000000000000000000000000..5ff718e66be7108eec6a97fdb8f789e66a0cf22e --- /dev/null +++ b/root/wander-train/stage5m6e_count_scope_successor.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import re +from pathlib import Path +from typing import Any,Sequence +from stage5m6a_oracle_successor_runtime import NUMBER_WORDS,_number +from stage5m6d_whole_system_successor import M6dStructuralReasoner,OracleM6dSuccessorRuntime +class M6eStructuralReasoner(M6dStructuralReasoner): + @staticmethod + def _object_counting(body:str,_choices:Sequence[str])->tuple[str|None,dict[str,Any]]: + prefix,_,_=body.partition("?");interrogative=re.search(r"\b(?:how|what|which)\b.+$",prefix,re.I);question=interrogative.group(0) if interrogative else "";statement=prefix[:interrogative.start()].rstrip(" .") if interrogative else prefix + inventory=re.sub(r"^.*?\b(?:have|has|contains?)\b","",statement,count=1,flags=re.I);segments=[v.strip(" .") for v in re.split(r",|\band\b",inventory,flags=re.I) if v.strip(" .")] + taxonomy={"animals":{"dog","cat","snake","chicken","frog","rabbit","pig","snail","horse","cow","goat","sheep","bird","duck","fish","hamster","mouse","rat","turtle","lizard"},"musical instruments":{"violin","viola","cello","guitar","piano","trumpet","clarinet","flute","drum","trombone","saxophone","harp","oboe","bassoon"},"instruments":{"violin","viola","cello","guitar","piano","trumpet","clarinet","flute","drum","trombone","saxophone","harp","oboe","bassoon"}} + requested=next((name for name in taxonomy if name in question.casefold()),None);parsed=[] + for segment in segments: + match=re.match(r"(?:an?|the)\s+(.+)$|(?:(\d+|"+"|".join(sorted(NUMBER_WORDS,key=len,reverse=True))+r")\s+)(.+)$",segment,re.I) + if not match:continue + count,noun=(1,match.group(1)) if match.group(1) is not None else (int(_number(match.group(2)) or 0),match.group(3));noun=re.sub(r"[^a-z -]","",noun.casefold()).strip();lemma=noun[:-2] if noun.endswith("es") and noun[:-2] in set().union(*taxonomy.values()) else noun[:-1] if noun.endswith("s") else noun;parsed.append((count,lemma)) + selected=parsed if requested is None else [(count,noun) for count,noun in parsed if noun in taxonomy[requested]];total=sum(count for count,_ in selected) + return (str(total),{"inventory":parsed,"question":question,"requested_taxonomy":requested,"selected":selected,"scope":"declarative_inventory_then_interrogative_taxonomy"}) if selected else (None,{"reason":"no_countable_objects","inventory":parsed,"question":question}) +class OracleM6eSuccessorRuntime(OracleM6dSuccessorRuntime): + def __init__(self,*,federation_manifest:Path,state_root:Path):super().__init__(federation_manifest=federation_manifest,state_root=state_root);self.reasoner=M6eStructuralReasoner() +__all__=["OracleM6eSuccessorRuntime","M6eStructuralReasoner"] diff --git a/root/wander-train/stage5m_persistent_substrate_bridge.py b/root/wander-train/stage5m_persistent_substrate_bridge.py new file mode 100644 index 0000000000000000000000000000000000000000..f69ce3544b15f95e4c3b385fd0d1093695372f05 --- /dev/null +++ b/root/wander-train/stage5m_persistent_substrate_bridge.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +"""Handle-only bridge between persistent HRR memory and an active Oracle Tree.""" + +from __future__ import annotations + +import copy +import json +from dataclasses import dataclass +from typing import Any, Mapping + +try: + from ensouled._substrate.oracle_memory import ( + OraclePersistentSubstrate, + SnapshotBundle, + SnapshotPayload, + canonical_json, + digest_value, + verify_snapshot_bundle, + ) +except ModuleNotFoundError: # Isolated flat deployment beside oracle_memory.py. + from oracle_memory import ( + OraclePersistentSubstrate, + SnapshotBundle, + SnapshotPayload, + canonical_json, + digest_value, + verify_snapshot_bundle, + ) + + +INGRESS_SCHEMA = "oracle-stage5m-persistent-substrate-ingress-v1" + + +@dataclass(frozen=True) +class MemoryRelease: + authorized: bool + reason: str + handle_id: str + roles: tuple[tuple[str, str], ...] + source_record_id: str | None + provenance: tuple[tuple[str, Any], ...] + + def role_map(self) -> dict[str, str]: + return dict(self.roles) + + +def _leaked_payload_values( + bundle: SnapshotBundle, + serialized: str, +) -> list[str]: + """Find lexical payload values without substring/key false positives. + + Very short control words can legitimately equal visible descriptor states + (for example ``unverified``), so the structural handle-only schema is the + authority for those values. Longer lexical values must not occur as exact + serialized JSON string tokens. + """ + return [ + value + for payload in bundle.payload_bank.entries + for role, value in payload.roles + if ( + role != "status" + and value + and len(value) >= 12 + and canonical_json(value) in serialized + ) + ] + + +def _handle_descriptor(handle) -> dict[str, Any]: + return { + "handle_id": handle.handle_id, + "bank": handle.bank, + "memory_digest": handle.memory_digest, + "provenance_digest": handle.provenance_digest, + "status": handle.status, + "confidence": handle.confidence, + "score": handle.score, + "skill_id": handle.skill_id, + "skill_revision": handle.skill_revision, + } + + +def build_tree_memory_ingress(bundle: SnapshotBundle) -> dict[str, Any]: + """Build the only persistent-memory object allowed to enter the Tree.""" + valid, reason = verify_snapshot_bundle(bundle) + if not valid: + raise ValueError(reason or "invalid persistent-memory snapshot") + snapshot = bundle.snapshot + ingress = { + "schema_version": INGRESS_SCHEMA, + "agent_id": snapshot.agent_id, + "snapshot_id": snapshot.snapshot_id, + "snapshot_digest": snapshot.snapshot_digest, + "state_version": snapshot.state_version, + "state_digest": snapshot.state_digest, + "query_digest": snapshot.query_digest, + "payload_bank_digest": snapshot.payload_bank_digest, + "candidate_handles": [ + _handle_descriptor(handle) + for handle in snapshot.handles + ], + "raw_payload_text_present": False, + "request_local_immutable_snapshot": True, + "malkuth_only_resolution": True, + } + serialized = canonical_json(ingress) + leaked = _leaked_payload_values(bundle, serialized) + if leaked: + raise RuntimeError( + f"persistent-memory Tree ingress leaked payloads: {leaked[:3]}" + ) + return ingress + + +def attach_memory_ingress( + tree_record: Mapping[str, Any], + bundle: SnapshotBundle, + *, + expected_agent_id: str | None = None, +) -> dict[str, Any]: + """Attach candidate handles and immutable snapshot provenance to a Tree.""" + record = copy.deepcopy(dict(tree_record)) + ingress = build_tree_memory_ingress(bundle) + if ( + expected_agent_id is not None + and ingress["agent_id"] != expected_agent_id + ): + raise ValueError("persistent substrate agent mismatch") + contract = record.setdefault("dual_rail_contract", {}) + content = contract.setdefault("content_rail", {}) + spine = contract.setdefault("stabilizing_spine", {}) + content["stage5m:content:persistent_memory_handles"] = [ + item["handle_id"] for item in ingress["candidate_handles"] + ] + spine["stage5m:spine:persistent_substrate"] = { + "agent_id": ingress["agent_id"], + "snapshot_id": ingress["snapshot_id"], + "snapshot_digest": ingress["snapshot_digest"], + "state_version": ingress["state_version"], + "state_digest": ingress["state_digest"], + "query_digest": ingress["query_digest"], + "payload_bank_digest": ingress["payload_bank_digest"], + "candidate_set_digest": digest_value( + ingress["candidate_handles"] + ), + "mutable_during_rollout": False, + "unbind_permission": "malkuth_only", + } + record["persistent_substrate_ingress"] = ingress + record.setdefault("payload_contract", {})[ + "persistent_memory_payload_bank_digest" + ] = ingress["payload_bank_digest"] + record["payload_contract"][ + "persistent_memory_raw_payload_text_present" + ] = False + + serialized = canonical_json(record) + leaked = _leaked_payload_values(bundle, serialized) + if leaked: + raise RuntimeError( + f"persistent-memory augmented Tree leaked payloads: {leaked[:3]}" + ) + return record + + +def _release_denied(handle_id: str, reason: str) -> MemoryRelease: + return MemoryRelease( + authorized=False, + reason=reason, + handle_id=handle_id, + roles=(), + source_record_id=None, + provenance=(), + ) + + +def release_memory_at_malkuth( + tree_record: Mapping[str, Any], + bundle: SnapshotBundle, + handle_id: str, + *, + malkuth_entered: bool, +) -> MemoryRelease: + """Resolve one snapshot handle only after Tree/spine authorization.""" + valid, reason = verify_snapshot_bundle(bundle) + if not valid: + return _release_denied(handle_id, reason or "snapshot_invalid") + ingress = tree_record.get("persistent_substrate_ingress") + if not isinstance(ingress, Mapping): + return _release_denied(handle_id, "persistent_ingress_absent") + snapshot = bundle.snapshot + expected = { + "agent_id": snapshot.agent_id, + "snapshot_id": snapshot.snapshot_id, + "snapshot_digest": snapshot.snapshot_digest, + "state_version": snapshot.state_version, + "state_digest": snapshot.state_digest, + "query_digest": snapshot.query_digest, + "payload_bank_digest": snapshot.payload_bank_digest, + } + for key, value in expected.items(): + if ingress.get(key) != value: + return _release_denied(handle_id, f"persistent_{key}_mismatch") + spine = tree_record.get("dual_rail_contract", {}).get( + "stabilizing_spine", + {}, + ) + substrate_spine = spine.get("stage5m:spine:persistent_substrate", {}) + if substrate_spine.get("agent_id") != snapshot.agent_id: + return _release_denied(handle_id, "persistent_spine_agent_mismatch") + if substrate_spine.get("snapshot_digest") != snapshot.snapshot_digest: + return _release_denied(handle_id, "persistent_spine_snapshot_mismatch") + if substrate_spine.get("mutable_during_rollout") is not False: + return _release_denied(handle_id, "persistent_snapshot_mutability_invalid") + if substrate_spine.get("unbind_permission") != "malkuth_only": + return _release_denied(handle_id, "persistent_unbind_permission_invalid") + root = tree_record.get("protected_roles", {}).get("root") + spine_root = spine.get("stage4:spine:root") + if root is not None and spine_root is not None and root != spine_root: + return _release_denied(handle_id, "protected_root_spine_mismatch") + handle_set = { + item.get("handle_id") + for item in ingress.get("candidate_handles", []) + } + content_handles = set( + tree_record.get("dual_rail_contract", {}) + .get("content_rail", {}) + .get("stage5m:content:persistent_memory_handles", []) + ) + if handle_id not in handle_set or handle_id not in content_handles: + return _release_denied(handle_id, "persistent_handle_not_in_tree") + if not malkuth_entered: + return _release_denied(handle_id, "premature_persistent_unbinding") + try: + payload = OraclePersistentSubstrate.resolve_snapshot_handle( + bundle, + handle_id, + ) + except (KeyError, ValueError, RuntimeError): + return _release_denied(handle_id, "persistent_payload_resolution_failed") + return MemoryRelease( + authorized=True, + reason="authorized", + handle_id=handle_id, + roles=payload.roles, + source_record_id=payload.source_record_id, + provenance=payload.provenance, + ) + + +def record_verified_tree_outcome( + store: OraclePersistentSubstrate, + *, + tree_record: Mapping[str, Any], + resolved_roles: Mapping[str, str], + response: str, + action: str, + verifier_outcome: str, + verifier_digest: str | None, + evidence_id: str, + source_record_id: str, + provenance: Mapping[str, Any], + event_key: str, +) -> str: + """Bind a completed Tree/action result back into long-term memory.""" + if verifier_outcome not in ("verified_success", "verified_failure", "unverified"): + raise ValueError(f"unsupported verifier_outcome: {verifier_outcome}") + if verifier_outcome != "unverified" and not verifier_digest: + raise ValueError("verified outcomes require verifier_digest") + spine = tree_record.get("dual_rail_contract", {}).get( + "stabilizing_spine", + {}, + ) + root = tree_record.get("protected_roles", {}).get("root") + spine_root = spine.get("stage4:spine:root") + if root is not None and spine_root is not None and root != spine_root: + raise ValueError("cannot persist outcome with a corrupted protected root") + status = { + "verified_success": "verified", + "verified_failure": "failed", + "unverified": "unverified", + }[verifier_outcome] + roles = { + role: value + for role, value in resolved_roles.items() + if role in { + "root", "goal", "situation", "source", "field", "transition", + "result", "warrant", "correction", + } + } + roles.update({ + "action": action, + "result": response, + "verifier": verifier_digest or "unverified", + "status": verifier_outcome, + }) + return store.bind_event( + bank="episodic", + roles=roles, + source_record_id=source_record_id, + provenance={ + **dict(provenance), + "tree_record_id": tree_record.get("id"), + "tree_record_digest": digest_value(tree_record), + "evidence_id": evidence_id, + }, + status=status, + confidence=1.0 if status == "verified" else 0.2 if status == "failed" else 0.5, + event_key=event_key, + metadata={ + "verifier_outcome": verifier_outcome, + "evidence_id": evidence_id, + }, + ) + + +def payload_contains_text(payload: SnapshotPayload, serialized: str) -> bool: + """Test helper used by causal leak controls.""" + return any(value in serialized for _, value in payload.roles if value) diff --git a/root/wander-train/train_stage2b_malkuth_decoder.py b/root/wander-train/train_stage2b_malkuth_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..41287e2e8d0c0c98c9dbbb039a348ac4f3b9f8b7 --- /dev/null +++ b/root/wander-train/train_stage2b_malkuth_decoder.py @@ -0,0 +1,882 @@ +#!/usr/bin/env python3 +"""Stage 2B: masked operator routing and contrastive Malkuth resolution. + +The decoder sees only the routed HRR spine state. Frozen carrier features may +select the operator and seed the content state, but there is no carrier-to- +answer bypass. This makes path, node, route, and protected-root ablations causal +interventions on the representation used for final resolution. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import random +import sys +import time +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from train_stage2a_role_ladder import ( # noqa: E402 + HRR_DIM, MODALITIES, OPERATIONS, OP_TO_IDX, bind, from_real, + lane_key, lexical_features, normalize, text_hrr, unitary, +) + +CANDIDATE_LEXICAL_DIM = 768 +CANDIDATE_HRR_DIM = 256 +CANDIDATE_ROLE_DIM = 256 +CANDIDATE_DIM = CANDIDATE_LEXICAL_DIM + CANDIDATE_HRR_DIM + CANDIDATE_ROLE_DIM + + +def candidate_projection() -> np.ndarray: + rng = np.random.default_rng(0x2B20260720) + matrix = rng.normal( + 0.0, 1.0 / math.sqrt(2 * HRR_DIM), + (2 * HRR_DIM, CANDIDATE_HRR_DIM), + ).astype(np.float32) + return normalize_rows(matrix.T).T + + +def candidate_role_projection() -> np.ndarray: + rng = np.random.default_rng(0x2B20260721) + matrix = rng.normal( + 0.0, 1.0 / math.sqrt(2 * HRR_DIM), + (2 * HRR_DIM, CANDIDATE_ROLE_DIM), + ).astype(np.float32) + return normalize_rows(matrix.T).T + + +def ordinal_role_state(text: str, record: dict[str, Any] | None) -> np.ndarray: + if record is None: + return np.zeros(HRR_DIM, dtype=np.complex64) + source = record["role_binding_map"]["source"] + field = record["role_binding_map"]["field"] + lowered = text.lower() + source_position = lowered.find(source.lower()) + field_position = lowered.find(field.lower()) + if source_position < 0 and field_position < 0: + return np.zeros(HRR_DIM, dtype=np.complex64) + ordered: list[str] = [] + if source_position >= 0 and field_position >= 0: + ordered = [source, field] if source_position < field_position else [field, source] + elif source_position >= 0: + ordered = [source] + else: + ordered = [field] + state = np.zeros(HRR_DIM, dtype=np.complex64) + for index, value in enumerate(ordered): + state += bind(unitary(f"stage2b:candidate-ordinal:{index}"), text_hrr(value)) + return normalize(state) + + +def candidate_embedding( + text: str, projection: np.ndarray, role_projection: np.ndarray, + record: dict[str, Any] | None, +) -> np.ndarray: + lexical = lexical_features(text, CANDIDATE_LEXICAL_DIM) + geometric = as_real(text_hrr(text)) @ projection + role_order = as_real(ordinal_role_state(text, record)) @ role_projection + combined = np.concatenate((lexical, geometric, role_order)).astype(np.float32) + return combined / max(float(np.linalg.norm(combined)), 1e-8) + + +def load_records(path: Path) -> list[dict[str, Any]]: + with path.open("r", encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def normalize_rows(values: np.ndarray) -> np.ndarray: + norms = np.linalg.norm(values, axis=1, keepdims=True) + return (values / np.maximum(norms, 1e-8)).astype(np.float32) + + +def as_real(vector: np.ndarray) -> np.ndarray: + return np.concatenate((vector.real, vector.imag)).astype(np.float32) + + +def fit_ridge(x: np.ndarray, y: np.ndarray, mask: np.ndarray, ridge: float) -> np.ndarray: + xt = x[mask].astype(np.float64) + yt = y[mask].astype(np.float64) + gram = xt @ xt.T + gram.flat[:: gram.shape[0] + 1] += ridge + alpha = np.linalg.solve(gram, yt) + return (xt.T @ alpha).astype(np.float32) + + +def binomial_tail(n: int, k: int, probability: float) -> float: + """Exact one-sided binomial tail, accumulated stably in log space.""" + if k <= 0: + return 1.0 + if k > n: + return 0.0 + if probability <= 0.0: + return 0.0 + if probability >= 1.0: + return 1.0 + log_p = math.log(probability) + log_q = math.log1p(-probability) + log_n_factorial = math.lgamma(n + 1) + terms = [ + log_n_factorial + - math.lgamma(successes + 1) + - math.lgamma(n - successes + 1) + + successes * log_p + + (n - successes) * log_q + for successes in range(k, n + 1) + ] + peak = max(terms) + return min(1.0, float(math.exp(peak) * sum(math.exp(term - peak) for term in terms))) + + +def wilson_interval(n: int, k: int, z: float = 1.959963984540054) -> list[float]: + p = k / max(n, 1) + denominator = 1 + z * z / max(n, 1) + center = (p + z * z / (2 * max(n, 1))) / denominator + radius = z * math.sqrt(p * (1 - p) / max(n, 1) + z * z / (4 * max(n, 1) ** 2)) / denominator + return [max(0.0, center - radius), min(1.0, center + radius)] + + +def macro_f1(truth: np.ndarray, predicted: np.ndarray, n_classes: int) -> float: + values = [] + for label in range(n_classes): + tp = int(np.sum((truth == label) & (predicted == label))) + fp = int(np.sum((truth != label) & (predicted == label))) + fn = int(np.sum((truth == label) & (predicted != label))) + precision = tp / max(tp + fp, 1) + recall = tp / max(tp + fn, 1) + values.append(2 * precision * recall / max(precision + recall, 1e-12)) + return float(np.mean(values)) + + +def mcnemar_one_sided(full_correct: np.ndarray, ablated_correct: np.ndarray) -> dict[str, Any]: + full_only = int(np.sum(full_correct & ~ablated_correct)) + ablated_only = int(np.sum(~full_correct & ablated_correct)) + discordant = full_only + ablated_only + p = binomial_tail(discordant, full_only, 0.5) if discordant else 1.0 + return { + "full_only_correct": full_only, + "ablated_only_correct": ablated_only, + "discordant": discordant, + "one_sided_p_full_better": p, + } + + +def holm_adjust(raw: dict[str, float]) -> dict[str, float]: + """Family-wise Holm adjustment, returned in the original label space.""" + ordered = sorted(raw, key=raw.get) + adjusted: dict[str, float] = {} + running = 0.0 + total = len(ordered) + for rank, label in enumerate(ordered): + running = max(running, (total - rank) * raw[label]) + adjusted[label] = min(1.0, running) + return adjusted + + +def load_carrier(records: list[dict[str, Any]], path: Path) -> dict[str, np.ndarray]: + archive = np.load(path, allow_pickle=False) + ids = [str(value) for value in archive["ids"]] + lookup = {value: index for index, value in enumerate(ids)} + expected = {record["id"] for record in records} + if set(lookup) != expected: + missing = sorted(expected - set(lookup)) + extra = sorted(set(lookup) - expected) + raise ValueError(f"carrier/corpus mismatch missing={missing[:3]} extra={extra[:3]}") + order = [lookup[record["id"]] for record in records] + result = { + "pooled": np.asarray(archive["pooled"][order], dtype=np.float32), + "comp_emb": np.asarray(archive["comp_emb"][order], dtype=np.float32), + "sephirah_logits": np.asarray(archive["sephirah_logits"][order], dtype=np.float32), + } + archive.close() + return result + + +def operator_features( + records: list[dict[str, Any]], carrier: dict[str, np.ndarray], hash_dim: int, include_lexical: bool, +) -> np.ndarray: + parts = [carrier["pooled"], carrier["comp_emb"], carrier["sephirah_logits"]] + features = normalize_rows(np.concatenate(parts, axis=1)) + if include_lexical: + lexical = np.stack([lexical_features(record["input_text"], hash_dim) for record in records]) + features = normalize_rows(np.concatenate((features, lexical), axis=1)) + return features + + +def operator_targets(records: list[dict[str, Any]]) -> np.ndarray: + targets = np.zeros((len(records), len(OPERATIONS)), dtype=np.float32) + for row, record in enumerate(records): + targets[row, OP_TO_IDX[record["operation"]]] = 1.0 + return targets + + +def evaluate_operator( + records: list[dict[str, Any]], scores: np.ndarray, split: str, +) -> tuple[dict[str, Any], np.ndarray, np.ndarray]: + mask = np.array([record["split"] == split for record in records]) + truth = np.array([OP_TO_IDX[record["operation"]] for record in records])[mask] + predicted = scores[mask].argmax(axis=1) + correct = int(np.sum(truth == predicted)) + n = len(truth) + confusion = np.zeros((len(OPERATIONS), len(OPERATIONS)), dtype=int) + for actual, guess in zip(truth, predicted): + confusion[actual, guess] += 1 + return ({ + "records": n, + "correct": correct, + "accuracy": correct / n, + "macro_f1": macro_f1(truth, predicted, len(OPERATIONS)), + "chance": 1 / len(OPERATIONS), + "binomial_p_one_sided": binomial_tail(n, correct, 1 / len(OPERATIONS)), + "accuracy_wilson_95": wilson_interval(n, correct), + "confusion": confusion.tolist(), + }, mask, predicted) + + +def route_library(records: list[dict[str, Any]]) -> dict[str, tuple[str, ...]]: + routes: dict[str, tuple[str, ...]] = {} + for record in records: + route = tuple(record["tree_route"]["nodes"]) + previous = routes.setdefault(record["operation"], route) + if previous != route: + raise ValueError(f"operation {record['operation']} has inconsistent routes") + return routes + + +def carrier_content(comp_real: np.ndarray, record: dict[str, Any], include_lexical: bool) -> np.ndarray: + content = normalize(from_real(comp_real)) + if include_lexical: + content = normalize(content + np.complex64(0.55) * text_hrr(record["input_text"])) + return content + + +def observable_spine( + record: dict[str, Any], content: np.ndarray, operation: str, + routes: dict[str, tuple[str, ...]], *, include_paths: bool = True, + include_nodes: bool = True, include_root: bool = True, + include_role_bindings: bool = True, omit_path: str | None = None, + omit_node: str | None = None, +) -> np.ndarray: + if include_role_bindings: + role_state = np.zeros(HRR_DIM, dtype=np.complex64) + for role_name in ("source", "field", "transition"): + value = record["role_binding_map"][role_name] + role_state += bind(unitary(f"stage2b:input-role:{role_name}"), text_hrr(value)) + content = normalize(content + np.complex64(0.85) * normalize(role_state)) + state = bind(unitary("stage2b:content-role"), content) + if include_root: + state += bind(unitary("stage2b:protected-root"), text_hrr(record["root_question"])) + nodes = routes[operation] + operator_key = unitary(f"stage2b:operator:{operation}") + if include_paths: + for source, target in zip(nodes, nodes[1:]): + name = f"{source}-{target}" + if name == omit_path: + continue + delta = normalize(bind(operator_key, bind(unitary(f"stage2b:path-delta:{name}"), content))) + folded = normalize(bind(unitary(f"tree-fold:path:{name}"), delta)) + state += np.complex64(0.72) * bind(lane_key("path", name), folded) + if include_nodes: + for node in nodes: + if node == omit_node: + continue + synthesis = normalize(bind(operator_key, bind(unitary(f"stage2b:node-synthesis:{node}"), content))) + folded = normalize(bind(unitary(f"tree-fold:node:{node}"), synthesis)) + state += np.complex64(0.72) * bind(lane_key("node", node), folded) + return as_real(normalize(state)) + + +def candidate_vectors(records: list[dict[str, Any]], field: str) -> tuple[np.ndarray, np.ndarray, list[list[str]]]: + candidates = np.empty((len(records), 4, CANDIDATE_DIM), dtype=np.float32) + correct = np.empty(len(records), dtype=int) + kinds: list[list[str]] = [] + projection = candidate_projection() + role_projection = candidate_role_projection() + index_field = "correct_decode_candidate_index" if field == "decode_candidates" else "correct_root_candidate_index" + for row, record in enumerate(records): + items = record[field] + kinds.append([item["kind"] for item in items]) + correct[row] = int(record[index_field]) + for column, item in enumerate(items): + role_record = record if field == "decode_candidates" else None + candidates[row, column] = candidate_embedding( + item["text"], projection, role_projection, role_record, + ) + candidates /= np.maximum(np.linalg.norm(candidates, axis=2, keepdims=True), 1e-8) + return candidates, correct, kinds + + +def continuation_vectors(records: list[dict[str, Any]]) -> tuple[np.ndarray, np.ndarray, list[list[str]]]: + """Build finite continuation alternatives without changing carrier inputs.""" + candidates = np.empty((len(records), 4, CANDIDATE_DIM), dtype=np.float32) + correct = np.empty(len(records), dtype=int) + kinds: list[list[str]] = [] + projection = candidate_projection() + role_projection = candidate_role_projection() + for row, record in enumerate(records): + literal = record["literal_decode"] + continuation = record["training_targets"]["conversational_continuation"] + if not continuation.startswith(literal): + raise ValueError(f"{record['id']}: conversational continuation does not preserve answer") + suffix = continuation[len(literal):] + topic_drift = next( + item["text"] for item in record["decode_candidates"] + if item["kind"] == "topic_drift" + ) + items = [ + {"kind": "correct_continuation", "text": continuation}, + {"kind": "answer_only_closure", "text": literal}, + { + "kind": "role_swap_continuation", + "text": record["minimal_role_swap"] + suffix, + }, + {"kind": "topic_drift_continuation", "text": topic_drift}, + ] + if len({item["text"] for item in items}) != 4: + raise ValueError(f"{record['id']}: continuation candidates are not unique") + seed = int.from_bytes( + hashlib.blake2b( + f"{record['id']}:continuation".encode("utf-8"), digest_size=8, + ).digest(), + "little", + ) + random.Random(seed).shuffle(items) + correct[row] = next( + index for index, item in enumerate(items) + if item["kind"] == "correct_continuation" + ) + kinds.append([item["kind"] for item in items]) + for column, item in enumerate(items): + candidates[row, column] = candidate_embedding( + item["text"], projection, role_projection, record, + ) + candidates /= np.maximum(np.linalg.norm(candidates, axis=2, keepdims=True), 1e-8) + return candidates, correct, kinds + + +def contrast_targets(candidates: np.ndarray, correct: np.ndarray) -> np.ndarray: + rows = np.arange(len(candidates)) + positive = candidates[rows, correct] + negative_sum = candidates.sum(axis=1) - positive + target = positive - negative_sum / (candidates.shape[1] - 1) + return normalize_rows(target) + + +def positive_only_targets(candidates: np.ndarray, correct: np.ndarray) -> np.ndarray: + """Baseline that never sees role-swap/operator/topic negative directions.""" + return candidates[np.arange(len(candidates)), correct].copy() + + +def rank_metrics( + records: list[dict[str, Any]], predictions: np.ndarray, candidates: np.ndarray, + correct_indices: np.ndarray, kinds: list[list[str]], mask: np.ndarray, + role_probe: bool, +) -> tuple[dict[str, Any], np.ndarray]: + pred = normalize_rows(predictions[mask]) + cand = candidates[mask] + correct = correct_indices[mask] + score = np.einsum("nd,nkd->nk", pred, cand) + guess = score.argmax(axis=1) + correct_flags = guess == correct + ordering = np.argsort(score, axis=1)[:, ::-1] + ranks = np.array([int(np.where(ordering[row] == correct[row])[0][0]) + 1 for row in range(len(correct))]) + margins = score[np.arange(len(correct)), correct] - np.max( + np.where(np.arange(4)[None, :] == correct[:, None], -np.inf, score), axis=1, + ) + result: dict[str, Any] = { + "records": len(correct), + "correct": int(correct_flags.sum()), + "accuracy": float(correct_flags.mean()), + "chance": 0.25, + "binomial_p_one_sided": binomial_tail(len(correct), int(correct_flags.sum()), 0.25), + "accuracy_wilson_95": wilson_interval(len(correct), int(correct_flags.sum())), + "mean_reciprocal_rank": float(np.mean(1.0 / ranks)), + "mean_correct_margin": float(np.mean(margins)), + "by_distractor_turns": {}, + } + masked_records = [record for record, keep in zip(records, mask) if keep] + for level in sorted({record["distractor_turn_count"] for record in masked_records}): + level_mask = np.array([record["distractor_turn_count"] == level for record in masked_records]) + result["by_distractor_turns"][str(level)] = { + "records": int(level_mask.sum()), + "accuracy": float(correct_flags[level_mask].mean()), + "mean_margin": float(margins[level_mask].mean()), + } + if role_probe: + masked_kinds = [value for value, keep in zip(kinds, mask) if keep] + role_indices = np.array([row.index("role_swap_minimal") for row in masked_kinds]) + role_wins = score[np.arange(len(correct)), correct] > score[np.arange(len(correct)), role_indices] + result["minimal_role_swap_pair_accuracy"] = float(role_wins.mean()) + result["minimal_role_swap_pair_correct"] = int(role_wins.sum()) + result["minimal_role_swap_binomial_p_one_sided"] = binomial_tail(len(role_wins), int(role_wins.sum()), 0.5) + return result, correct_flags + + +def build_spines( + records: list[dict[str, Any]], carrier: dict[str, np.ndarray], operations: list[str], + routes: dict[str, tuple[str, ...]], include_lexical: bool, variant: str, +) -> np.ndarray: + result = np.empty((len(records), 2 * HRR_DIM), dtype=np.float32) + for row, (record, operation) in enumerate(zip(records, operations)): + content = carrier_content(carrier["comp_emb"][row], record, include_lexical) + kwargs = { + "include_paths": variant not in ("no_paths", "flat_no_lanes"), + "include_nodes": variant not in ("no_nodes", "flat_no_lanes"), + "include_root": variant not in ( + "no_protected_root", "protected_root_late_fusion", + ), + "include_role_bindings": variant != "no_role_bindings", + "omit_path": variant.split(":", 1)[1] if variant.startswith("drop_path:") else None, + "omit_node": variant.split(":", 1)[1] if variant.startswith("drop_node:") else None, + } + if variant == "wrong_route": + operation = OPERATIONS[(OP_TO_IDX[operation] + 1) % len(OPERATIONS)] + result[row] = observable_spine(record, content, operation, routes, **kwargs) + return result + + +def protected_root_inputs( + records: list[dict[str, Any]], spines: np.ndarray, include_protected_root: bool, +) -> np.ndarray: + if not include_protected_root: + return normalize_rows(spines) + roots = np.stack([as_real(text_hrr(record["root_question"])) for record in records]) + # Malkuth unbinds the invariant carrier exactly; a small transport term + # retains accumulated context without asking the noisy bundle to recreate root. + return normalize_rows(roots + np.float32(0.15) * spines) + + +def run_mode( + mode_name: str, records: list[dict[str, Any]], carrier: dict[str, np.ndarray], + routes: dict[str, tuple[str, ...]], train_mask: np.ndarray, + dev_mask: np.ndarray, test_mask: np.ndarray, decode_candidates: np.ndarray, + decode_correct: np.ndarray, decode_kinds: list[list[str]], + continuation_candidates: np.ndarray, continuation_correct: np.ndarray, + continuation_kinds: list[list[str]], root_candidates: np.ndarray, + root_correct: np.ndarray, root_kinds: list[list[str]], ridge: float, hash_dim: int, +) -> tuple[dict[str, Any], dict[str, np.ndarray]]: + include_lexical = mode_name == "carrier_plus_masked_lexical" + op_features = operator_features(records, carrier, hash_dim, include_lexical) + op_target = operator_targets(records) + op_weights = fit_ridge(op_features, op_target, train_mask, ridge) + op_scores = op_features @ op_weights + dev_op, _, _ = evaluate_operator(records, op_scores, "dev") + test_op, _, test_predictions = evaluate_operator(records, op_scores, "test") + + all_predicted = op_scores.argmax(axis=1) + predicted_operations = [OPERATIONS[index] for index in all_predicted] + gold_operations = [record["operation"] for record in records] + full_gold = build_spines(records, carrier, gold_operations, routes, include_lexical, "full") + decode_target = contrast_targets(decode_candidates, decode_correct) + positive_only_target = positive_only_targets(decode_candidates, decode_correct) + continuation_target = contrast_targets(continuation_candidates, continuation_correct) + positive_only_continuation_target = positive_only_targets( + continuation_candidates, continuation_correct, + ) + root_target = contrast_targets(root_candidates, root_correct) + decode_weights = fit_ridge(full_gold, decode_target, train_mask, ridge) + positive_only_weights = fit_ridge(full_gold, positive_only_target, train_mask, ridge) + continuation_weights = fit_ridge(full_gold, continuation_target, train_mask, ridge) + positive_only_continuation_weights = fit_ridge( + full_gold, positive_only_continuation_target, train_mask, ridge, + ) + root_weights = fit_ridge( + protected_root_inputs(records, full_gold, True), root_target, train_mask, ridge, + ) + + variants = ( + "full", "no_paths", "no_nodes", "flat_no_lanes", + "no_protected_root", "no_role_bindings", "wrong_route", + "protected_root_late_fusion", + ) + decode_results: dict[str, Any] = {} + continuation_results: dict[str, Any] = {} + root_results: dict[str, Any] = {} + decode_flags: dict[str, np.ndarray] = {} + continuation_flags: dict[str, np.ndarray] = {} + root_flags: dict[str, np.ndarray] = {} + for variant in variants: + spines = build_spines(records, carrier, predicted_operations, routes, include_lexical, variant) + decode_metrics, decode_correct_flags = rank_metrics( + records, spines @ decode_weights, decode_candidates, decode_correct, + decode_kinds, test_mask, True, + ) + continuation_metrics, continuation_correct_flags = rank_metrics( + records, spines @ continuation_weights, continuation_candidates, + continuation_correct, continuation_kinds, test_mask, False, + ) + root_metrics, root_correct_flags = rank_metrics( + records, + protected_root_inputs(records, spines, variant != "no_protected_root") @ root_weights, + root_candidates, root_correct, + root_kinds, test_mask, False, + ) + decode_results[variant] = decode_metrics + continuation_results[variant] = continuation_metrics + root_results[variant] = root_metrics + decode_flags[variant] = decode_correct_flags + continuation_flags[variant] = continuation_correct_flags + root_flags[variant] = root_correct_flags + + # This is an objective ablation, not a representational ablation: it uses + # the identical full predicted spine but a head fitted only to positives. + predicted_full = build_spines( + records, carrier, predicted_operations, routes, include_lexical, "full", + ) + positive_only_metrics, positive_only_flags = rank_metrics( + records, predicted_full @ positive_only_weights, decode_candidates, + decode_correct, decode_kinds, test_mask, True, + ) + positive_only_continuation_metrics, positive_only_continuation_flags = rank_metrics( + records, predicted_full @ positive_only_continuation_weights, + continuation_candidates, continuation_correct, continuation_kinds, + test_mask, False, + ) + + individual_interventions: dict[str, Any] = {} + if include_lexical: + full_decode_predictions = predicted_full @ decode_weights + full_continuation_predictions = predicted_full @ continuation_weights + route_nodes = [routes[operation] for operation in predicted_operations] + route_paths = [ + tuple(f"{source}-{target}" for source, target in zip(nodes, nodes[1:])) + for nodes in route_nodes + ] + intervention_spine_cache: dict[str, np.ndarray] = {} + + def evaluate_family( + labels: list[str], active_for: Any, variant_for: Any, + ) -> dict[str, Any]: + family: dict[str, Any] = {} + for label in labels: + active_mask = test_mask & np.array([ + bool(active_for(row, label)) for row in range(len(records)) + ]) + if not np.any(active_mask): + continue + variant = variant_for(label) + if variant not in intervention_spine_cache: + intervention_spine_cache[variant] = build_spines( + records, carrier, predicted_operations, routes, include_lexical, + variant, + ) + ablated = intervention_spine_cache[variant] + ablated_decode_predictions = np.zeros( + (len(records), CANDIDATE_DIM), dtype=np.float32, + ) + ablated_continuation_predictions = np.zeros_like( + ablated_decode_predictions, + ) + ablated_decode_predictions[active_mask] = ( + ablated[active_mask] @ decode_weights + ) + ablated_continuation_predictions[active_mask] = ( + ablated[active_mask] @ continuation_weights + ) + full_decode, full_decode_flags = rank_metrics( + records, full_decode_predictions, decode_candidates, + decode_correct, decode_kinds, active_mask, True, + ) + ablated_decode, ablated_decode_flags = rank_metrics( + records, ablated_decode_predictions, decode_candidates, + decode_correct, decode_kinds, active_mask, True, + ) + full_continuation, full_continuation_flags = rank_metrics( + records, full_continuation_predictions, + continuation_candidates, continuation_correct, + continuation_kinds, active_mask, False, + ) + ablated_continuation, ablated_continuation_flags = rank_metrics( + records, ablated_continuation_predictions, continuation_candidates, + continuation_correct, continuation_kinds, active_mask, False, + ) + family[label] = { + "active_test_records": int(active_mask.sum()), + "decode": { + "full_accuracy": full_decode["accuracy"], + "ablated_accuracy": ablated_decode["accuracy"], + "delta_full_minus_ablation": ( + full_decode["accuracy"] - ablated_decode["accuracy"] + ), + "mcnemar": mcnemar_one_sided( + full_decode_flags, ablated_decode_flags, + ), + }, + "continuation": { + "full_accuracy": full_continuation["accuracy"], + "ablated_accuracy": ablated_continuation["accuracy"], + "delta_full_minus_ablation": ( + full_continuation["accuracy"] + - ablated_continuation["accuracy"] + ), + "mcnemar": mcnemar_one_sided( + full_continuation_flags, ablated_continuation_flags, + ), + }, + } + for task in ("decode", "continuation"): + adjusted = holm_adjust({ + label: result[task]["mcnemar"]["one_sided_p_full_better"] + for label, result in family.items() + }) + for label, value in adjusted.items(): + family[label][task]["mcnemar"]["holm_adjusted_p_full_better"] = value + return family + + path_labels = sorted({path for paths in route_paths for path in paths}) + node_labels = sorted({node for nodes in route_nodes for node in nodes}) + individual_interventions = { + "path_lanes": evaluate_family( + path_labels, + lambda row, label: label in route_paths[row], + lambda label: f"drop_path:{label}", + ), + "node_lanes": evaluate_family( + node_labels, + lambda row, label: label in route_nodes[row], + lambda label: f"drop_node:{label}", + ), + "operator_routes": evaluate_family( + list(OPERATIONS), + lambda row, label: predicted_operations[row] == label, + lambda _label: "wrong_route", + ), + } + + causal = {} + for variant in variants[1:]: + if variant == "protected_root_late_fusion": + continue + causal[variant] = { + "decode_accuracy_delta_full_minus_ablation": ( + decode_results["full"]["accuracy"] - decode_results[variant]["accuracy"] + ), + "decode_mcnemar": mcnemar_one_sided(decode_flags["full"], decode_flags[variant]), + "continuation_accuracy_delta_full_minus_ablation": ( + continuation_results["full"]["accuracy"] + - continuation_results[variant]["accuracy"] + ), + "continuation_mcnemar": mcnemar_one_sided( + continuation_flags["full"], continuation_flags[variant], + ), + "root_accuracy_delta_full_minus_ablation": ( + root_results["full"]["accuracy"] - root_results[variant]["accuracy"] + ), + "root_mcnemar": mcnemar_one_sided(root_flags["full"], root_flags[variant]), + } + causal["no_contrastive_negatives"] = { + "decode_accuracy_delta_full_minus_ablation": ( + decode_results["full"]["accuracy"] - positive_only_metrics["accuracy"] + ), + "decode_mcnemar": mcnemar_one_sided( + decode_flags["full"], positive_only_flags, + ), + "continuation_accuracy_delta_full_minus_ablation": ( + continuation_results["full"]["accuracy"] + - positive_only_continuation_metrics["accuracy"] + ), + "continuation_mcnemar": mcnemar_one_sided( + continuation_flags["full"], positive_only_continuation_flags, + ), + } + corrected_name = "protected_root_late_fusion" + post_hoc_correction = { + "name": corrected_name, + "confirmatory": False, + "reason": ( + "The primary result showed that superposing the protected root into the answer " + "bundle harmed resolution. This intervention keeps root on its invariant carrier " + "and exposes it only to the root head at late Malkuth fusion." + ), + "decode_accuracy_delta_correction_minus_primary": ( + decode_results[corrected_name]["accuracy"] - decode_results["full"]["accuracy"] + ), + "decode_mcnemar_correction_better": mcnemar_one_sided( + decode_flags[corrected_name], decode_flags["full"], + ), + "continuation_accuracy_delta_correction_minus_primary": ( + continuation_results[corrected_name]["accuracy"] + - continuation_results["full"]["accuracy"] + ), + "continuation_mcnemar_correction_better": mcnemar_one_sided( + continuation_flags[corrected_name], continuation_flags["full"], + ), + "root_accuracy_delta_correction_minus_primary": ( + root_results[corrected_name]["accuracy"] - root_results["full"]["accuracy"] + ), + "root_mcnemar_correction_better": mcnemar_one_sided( + root_flags[corrected_name], root_flags["full"], + ), + } + return ({ + "feature_mode": mode_name, + "operator_routing": {"dev": dev_op, "test": test_op}, + "malkuth_decode": decode_results, + "conversational_continuation": continuation_results, + "root_retention": root_results, + "training_objective_baselines": { + "no_contrastive_negatives": { + "malkuth_decode": positive_only_metrics, + "conversational_continuation": positive_only_continuation_metrics, + }, + }, + "individual_interventions": individual_interventions, + "post_hoc_architectural_correction": post_hoc_correction, + "causal_ablations": causal, + "test_predicted_operation_counts": dict(Counter(OPERATIONS[index] for index in test_predictions)), + }, { + "operator_weights": op_weights, + "decode_weights": decode_weights, + "continuation_weights": continuation_weights, + "root_weights": root_weights, + }) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--corpus", type=Path, required=True) + parser.add_argument("--carrier-features", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--metrics", type=Path, required=True) + parser.add_argument("--ridge", type=float, default=0.20) + parser.add_argument("--hash-dim", type=int, default=768) + args = parser.parse_args() + started = time.perf_counter() + records = load_records(args.corpus) + carrier = load_carrier(records, args.carrier_features) + routes = route_library(records) + train_mask = np.array([record["split"] == "train" for record in records]) + dev_mask = np.array([record["split"] == "dev" for record in records]) + test_mask = np.array([record["split"] == "test" for record in records]) + decode_candidates, decode_correct, decode_kinds = candidate_vectors(records, "decode_candidates") + continuation_candidates, continuation_correct, continuation_kinds = continuation_vectors(records) + root_candidates, root_correct, root_kinds = candidate_vectors(records, "root_candidates") + + carrier_only, _ = run_mode( + "carrier_only", records, carrier, routes, train_mask, dev_mask, test_mask, + decode_candidates, decode_correct, decode_kinds, continuation_candidates, + continuation_correct, continuation_kinds, root_candidates, root_correct, root_kinds, + args.ridge, args.hash_dim, + ) + combined, weights = run_mode( + "carrier_plus_masked_lexical", records, carrier, routes, train_mask, dev_mask, test_mask, + decode_candidates, decode_correct, decode_kinds, continuation_candidates, + continuation_correct, continuation_kinds, root_candidates, root_correct, root_kinds, + args.ridge, args.hash_dim, + ) + + full = combined["malkuth_decode"]["full"] + continuation_full = combined["conversational_continuation"]["full"] + root_full = combined["root_retention"]["full"] + metrics = { + "schema_version": "oracle-phase2b-malkuth-v1", + "records": len(records), + "train_records": int(train_mask.sum()), + "dev_records": int(dev_mask.sum()), + "test_records": int(test_mask.sum()), + "carrier_checkpoint": str(args.carrier_features.resolve()), + "ridge": args.ridge, + "modes": { + "carrier_only": carrier_only, + "carrier_plus_masked_lexical": combined, + }, + "elapsed_seconds": time.perf_counter() - started, + "gates": { + "masked_operator_above_chance": ( + combined["operator_routing"]["test"]["accuracy"] > 1 / len(OPERATIONS) + and combined["operator_routing"]["test"]["binomial_p_one_sided"] < 0.01 + ), + "contrastive_decode_above_chance": full["accuracy"] > 0.25 + and full["binomial_p_one_sided"] < 0.01, + "minimal_role_swap_above_chance": full["minimal_role_swap_pair_accuracy"] > 0.5 + and full["minimal_role_swap_binomial_p_one_sided"] < 0.01, + "finite_continuation_above_chance": continuation_full["accuracy"] > 0.25 + and continuation_full["binomial_p_one_sided"] < 0.01, + "root_retention_above_chance": root_full["accuracy"] > 0.25 + and root_full["binomial_p_one_sided"] < 0.01, + "spine_has_positive_causal_effect": any( + value["decode_accuracy_delta_full_minus_ablation"] > 0 + and value["decode_mcnemar"]["one_sided_p_full_better"] < 0.05 + for name, value in combined["causal_ablations"].items() + if name != "no_contrastive_negatives" + ), + "aggregate_paths_have_positive_effect": ( + combined["causal_ablations"]["no_paths"] + ["decode_accuracy_delta_full_minus_ablation"] > 0 + and combined["causal_ablations"]["no_paths"] + ["decode_mcnemar"]["one_sided_p_full_better"] < 0.05 + ), + "aggregate_nodes_have_positive_effect": ( + combined["causal_ablations"]["no_nodes"] + ["decode_accuracy_delta_full_minus_ablation"] > 0 + and combined["causal_ablations"]["no_nodes"] + ["decode_mcnemar"]["one_sided_p_full_better"] < 0.05 + ), + "some_individual_path_lane_has_positive_effect": any( + result["decode"]["delta_full_minus_ablation"] > 0 + and result["decode"]["mcnemar"]["holm_adjusted_p_full_better"] < 0.05 + for result in combined["individual_interventions"]["path_lanes"].values() + ), + "some_individual_node_lane_has_positive_effect": any( + result["decode"]["delta_full_minus_ablation"] > 0 + and result["decode"]["mcnemar"]["holm_adjusted_p_full_better"] < 0.05 + for result in combined["individual_interventions"]["node_lanes"].values() + ), + "some_operator_route_has_positive_effect": any( + result["decode"]["delta_full_minus_ablation"] > 0 + and result["decode"]["mcnemar"]["holm_adjusted_p_full_better"] < 0.05 + for result in combined["individual_interventions"]["operator_routes"].values() + ), + "role_bindings_have_positive_effect": ( + combined["causal_ablations"]["no_role_bindings"] + ["decode_accuracy_delta_full_minus_ablation"] > 0 + and combined["causal_ablations"]["no_role_bindings"] + ["decode_mcnemar"]["one_sided_p_full_better"] < 0.05 + ), + "protected_root_has_positive_effect": ( + combined["causal_ablations"]["no_protected_root"] + ["root_accuracy_delta_full_minus_ablation"] > 0 + and combined["causal_ablations"]["no_protected_root"] + ["root_mcnemar"]["one_sided_p_full_better"] < 0.05 + ), + "contrastive_negatives_have_positive_effect": ( + combined["causal_ablations"]["no_contrastive_negatives"] + ["decode_accuracy_delta_full_minus_ablation"] > 0 + and combined["causal_ablations"]["no_contrastive_negatives"] + ["decode_mcnemar"]["one_sided_p_full_better"] < 0.05 + ), + "open_ended_generation_evaluated": False, + }, + "claim_boundary": ( + "This stage evaluates masked operator selection and finite-candidate Malkuth resolution. " + "It does not evaluate free-form generation or human conversational quality." + ), + } + args.metrics.parent.mkdir(parents=True, exist_ok=True) + args.metrics.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8") + args.out.parent.mkdir(parents=True, exist_ok=True) + with args.out.open("wb") as handle: + np.savez_compressed( + handle, + schema_version=np.array("oracle-phase2b-malkuth-checkpoint-v1"), + operator_weights=weights["operator_weights"], + decode_weights=weights["decode_weights"], + continuation_weights=weights["continuation_weights"], + root_weights=weights["root_weights"], + ridge=np.array(args.ridge), + hash_dim=np.array(args.hash_dim), + operations=np.asarray(OPERATIONS), + modalities=np.asarray(MODALITIES), + ) + print(json.dumps(metrics, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/root/wander-train/train_stage2d_factorized_specialist.py b/root/wander-train/train_stage2d_factorized_specialist.py new file mode 100644 index 0000000000000000000000000000000000000000..587994d63897ab90ae4e68b9c9d0eb75fb6098d4 --- /dev/null +++ b/root/wander-train/train_stage2d_factorized_specialist.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +"""Factorized Stage 2D semantic canonicalizer and variable role binder. + +The v1 shared random-feature bottleneck overfit topic identity. This revision +keeps the two mathematical problems separate: operations are classified from +the isolated final request, while roles are scored conditionally by locating +candidate fillers and representing the context in which each filler occurs. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import time +from collections import Counter +from pathlib import Path +from typing import Any + +import numpy as np + +from train_stage2a_role_ladder import OPERATIONS, OP_TO_IDX, lexical_features +from train_stage2c_direct_tree import fit_ridge_primal, load_records, normalize_rows +from train_stage2d_semantic_specialist import operation_metrics, operation_targets + + +RIDGES = (0.01, 0.05, 0.20, 1.00, 5.00, 20.00) +OPERATOR_MODES = ( + "instruction_lexical", + "instruction_lexical_curriculum", + "instruction_carrier", + "instruction_anchor", + "instruction_relational", + "lexical_plus_carrier", + "lexical_plus_anchor", + "lexical_plus_relational", + "lexical_plus_all_frozen", +) +ROLE_FEATURE_DIM = 2048 +TOKEN_RE = re.compile(r"[a-z0-9']+|[^\w\s]", re.I) + +OPERATOR_CURRICULUM = { + "analogy": ( + "map a preserved relational pattern across different participants", + "find the structural correspondence rather than shared vocabulary", + "complete a proportion while keeping each role in place", + "identify the invariant arrangement that makes two scenes comparable", + "transfer the syntax of one relation into another domain", + "locate a relational rhyme between unlike objects", + "ask what commutes when names change but positions remain", + "use one pair as the template for a matching pair", + "carry a relation as relational syntax even when every participant name changes", + ), + "mechanism": ( + "trace a causal process through intermediate state changes", + "open the black box between an initiating condition and its result", + "state the dependency chain whose interruption stops the outcome", + "distinguish a forward mechanism from a backward success story", + "make every transition independently observable and falsifiable", + "explain what changes first and what that enables next", + "separate causal transmission from temporal coincidence", + "identify the operative sequence connecting cause to endpoint", + "run an explanation forward through the process rather than reconstructing it backward from a finished outcome", + ), + "contrast": ( + "draw the functional boundary between two noninterchangeable roles", + "locate the asymmetry on which the conclusion depends", + "show where substitution of one participant for another fails", + "make a difference operational rather than merely verbal", + "preserve a decisive non-equivalence despite surface resemblance", + "test the account by letting the participants trade jobs", + "separate two contributions without collapsing their functions", + "identify which distinction carries explanatory weight", + "keep a functional difference intact when similar wording tempts an invalid exchange", + ), + "synthesis": ( + "integrate two partial truths into one mutually constrained whole", + "keep both pressures active rather than averaging them away", + "derive the new organization created by continued tension", + "show what interaction explains beyond mere juxtaposition", + "preserve distinguishable contributions in a joint result", + "compose incomplete explanations into a larger account", + "find the constraint that appears only when both sides remain", + "make neither view win by deleting the other", + "build a whole from distinguishable parts whose result depends on both remaining active", + ), + "return_to_root": ( + "return to the opening question after an informative detour", + "remove irrelevant branches and answer the original obligation", + "recover the trunk of the inquiry from later context", + "use intervening evidence to settle the first concern directly", + "close the loop rather than summarize every remark", + "distinguish useful context from conversational drift", + "reconstruct what the exchange was initially required to resolve", + "bring a branched discussion back to its first coordinate", + "recover the first obligation after later material has displaced attention", + ), + "conversational_continuation": ( + "answer the present question and derive one precise next inquiry", + "resolve this turn before opening an evidence-bearing next door", + "continue with a discriminating follow-up rather than a generic invitation", + "identify the uncertainty that could change confidence after answering", + "leave one warranted line of inquiry open after substantive closure", + "turn a complete answer toward its weakest remaining dependency", + "advance dialogue without replacing the owed answer with a question", + "form the next conversational move from the current resolution", + "finish the answer and expose one consequential uncertainty for the following turn", + ), + "clarification_repair": ( + "repair exchanged source and field roles before drawing a conclusion", + "disambiguate who acts what situates and what changes", + "correct a referential assignment hidden by fluent wording", + "restore the actor context transition triple before answering", + "diagnose a role binding fault rather than accepting a plausible sentence", + "rebind each filler to its intended function", + "stop and resolve which participant is entrant and which is setting", + "treat a source field swap as a different question", + "resolve an ambiguous assignment of actor setting and change before interpreting the result", + ), +} + + +def instruction_id(surface: str) -> str: + digest = hashlib.blake2b(surface.encode("utf-8"), digest_size=12).hexdigest() + return f"p2d-instruction-{digest}" + + +def adaptive_ridge(x: np.ndarray, y: np.ndarray, ridge: float) -> np.ndarray: + if x.shape[1] <= x.shape[0]: + return fit_ridge_primal(x, y, ridge) + x64 = x.astype(np.float64) + y64 = y.astype(np.float64) + gram = x64 @ x64.T + gram.flat[::gram.shape[0] + 1] += ridge + dual = np.linalg.solve(gram, y64) + return (x64.T @ dual).astype(np.float32) + + +def load_instruction_archive( + observation_records: list[dict[str, Any]], path: Path, +) -> dict[str, np.ndarray]: + archive = np.load(path, allow_pickle=False) + ids = [str(value) for value in archive["ids"]] + lookup = {value: index for index, value in enumerate(ids)} + expected = {record["id"] for record in observation_records} + if set(lookup) != expected: + archive.close() + raise ValueError("instruction feature IDs do not match observation corpus") + order = [lookup[record["id"]] for record in observation_records] + names = ( + "carrier_pooled", "carrier_comp_emb", "carrier_sephirah_logits", + "anchor_block4", "anchor_block5", "anchor_final", + "relational_trunk_final", "relational_hidden", "relational_experience", + "relational_sephirah_logits", + ) + result = {name: np.asarray(archive[name][order], dtype=np.float32) for name in names} + result["corpus_sha256"] = np.asarray(str(archive["corpus_sha256"])) + archive.close() + for name, value in result.items(): + if value.dtype.kind in "fci" and not np.isfinite(value).all(): + raise FloatingPointError(f"non-finite instruction feature: {name}") + return result + + +def group_normalized(*groups: np.ndarray) -> np.ndarray: + return normalize_rows(np.concatenate([normalize_rows(group) for group in groups], axis=1)) + + +def instruction_mode_features( + observations: list[dict[str, Any]], archive: dict[str, np.ndarray], mode: str, +) -> np.ndarray: + lexical = np.stack([ + lexical_features(record["instruction_surface"], 4096) for record in observations + ]).astype(np.float32) + carrier = np.concatenate([ + archive["carrier_pooled"], archive["carrier_comp_emb"], + archive["carrier_sephirah_logits"], + ], axis=1) + anchor = np.concatenate([ + archive["anchor_block4"], archive["anchor_block5"], archive["anchor_final"], + ], axis=1) + relational = np.concatenate([ + archive["relational_trunk_final"], archive["relational_hidden"], + archive["relational_experience"], archive["relational_sephirah_logits"], + ], axis=1) + if mode in ("instruction_lexical", "instruction_lexical_curriculum"): + return normalize_rows(lexical) + if mode == "instruction_carrier": + return normalize_rows(carrier) + if mode == "instruction_anchor": + return normalize_rows(anchor) + if mode == "instruction_relational": + return normalize_rows(relational) + if mode == "lexical_plus_carrier": + return group_normalized(lexical, carrier) + if mode == "lexical_plus_anchor": + return group_normalized(lexical, anchor) + if mode == "lexical_plus_relational": + return group_normalized(lexical, relational) + if mode == "lexical_plus_all_frozen": + return group_normalized(lexical, carrier, anchor, relational) + raise ValueError(mode) + + +def expand_instruction_features( + records: list[dict[str, Any]], observations: list[dict[str, Any]], features: np.ndarray, +) -> np.ndarray: + lookup = {record["id"]: features[index] for index, record in enumerate(observations)} + return np.stack([lookup[instruction_id(record["instruction_surface"])] for record in records]) + + +def value_context(clause: str, value: str, radius: int = 5) -> tuple[list[str], list[str], int, int]: + lower = clause.lower() + start = lower.find(value.lower()) + if start < 0: + return [], [], -1, 0 + end = start + len(value) + before = TOKEN_RE.findall(lower[:start])[-radius:] + after = TOKEN_RE.findall(lower[end:])[:radius] + return before, after, start, lower.count(value.lower()) + + +def candidate_role_features(clause: str, candidate: dict[str, str]) -> np.ndarray: + pseudo_parts = [] + positions = {} + counts = {} + for role in ("source", "field", "transition"): + before, after, position, count = value_context(clause, candidate[role]) + positions[role] = position + counts[role] = count + left = " ".join(f"left{offset}:{token}" for offset, token in enumerate(reversed(before), 1)) + right = " ".join(f"right{offset}:{token}" for offset, token in enumerate(after, 1)) + pseudo_parts.append( + f"assigned_role_{role} occurrence_{min(count, 2)} {left} filler_{role} {right}" + ) + lexical = lexical_features(" | ".join(pseudo_parts), ROLE_FEATURE_DIM) + length = max(len(clause), 1) + numeric = np.zeros(20, dtype=np.float32) + for index, role in enumerate(("source", "field", "transition")): + numeric[index] = positions[role] >= 0 + numeric[3 + index] = positions[role] / length if positions[role] >= 0 else -1.0 + numeric[6 + index] = min(counts[role], 2) / 2.0 + numeric[9] = positions["source"] < positions["field"] if min(positions.values()) >= 0 else 0 + numeric[10] = positions["field"] < positions["transition"] if min(positions.values()) >= 0 else 0 + numeric[11] = positions["source"] < positions["transition"] if min(positions.values()) >= 0 else 0 + # Quantized position indicators preserve template geometry without encoding filler identity. + for index, role in enumerate(("source", "field", "transition")): + if positions[role] >= 0: + numeric[12 + index] = min(int(4 * positions[role] / length), 3) / 3.0 + numeric[15] = sum(position >= 0 for position in positions.values()) / 3.0 + numeric[16] = len(candidate["source"]) / max(len(clause), 1) + numeric[17] = len(candidate["field"]) / max(len(clause), 1) + numeric[18] = len(candidate["transition"]) / max(len(clause), 1) + numeric[19] = 1.0 + value = np.concatenate([lexical, numeric]) + return value / max(float(np.linalg.norm(value)), 1e-8) + + +def role_pair_dataset( + records: list[dict[str, Any]], split: str | None = None, +) -> tuple[np.ndarray, np.ndarray, list[tuple[int, int]]]: + features, targets, index = [], [], [] + for row, record in enumerate(records): + if split is not None and record["split"] != split: + continue + for candidate_index, candidate in enumerate(record["role_candidates"]): + features.append(candidate_role_features(record["role_clause"], candidate["map"])) + targets.append(1.0 if candidate_index == 0 else -1.0) + index.append((row, candidate_index)) + return np.stack(features), np.asarray(targets, dtype=np.float32)[:, None], index + + +def role_metrics_from_scores( + records: list[dict[str, Any]], scores: np.ndarray, split: str, +) -> dict[str, Any]: + rows = [index for index, record in enumerate(records) if record["split"] == split] + local = scores[rows] + choices = local.argmax(axis=1) + swap_wins = local[:, 0] > local[:, 1] + by_depth = {} + for depth in (0, 2, 4, 8): + mask = np.asarray([records[row]["distractor_turn_count"] == depth for row in rows]) + by_depth[str(depth)] = { + "four_way_accuracy": float(np.mean(choices[mask] == 0)), + "correct_over_exact_swap": float(np.mean(swap_wins[mask])), + } + return { + "records": len(rows), + "four_way_accuracy": float(np.mean(choices == 0)), + "correct_over_exact_role_swap": float(np.mean(swap_wins)), + "choice_counts": dict(Counter( + records[row]["role_candidates"][choice]["kind"] + for row, choice in zip(rows, choices) + )), + "by_distractor_turns": by_depth, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--corpus", required=True, type=Path) + parser.add_argument("--instruction-corpus", required=True, type=Path) + parser.add_argument("--instruction-features", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--metrics", required=True, type=Path) + args = parser.parse_args() + started = time.perf_counter() + records = load_records(args.corpus) + observations = load_records(args.instruction_corpus) + if {record["split"] for record in records} != {"train", "dev"}: + raise ValueError("factorized specialist accepts train/dev only") + archive = load_instruction_archive(observations, args.instruction_features) + observation_train = np.asarray([record["split"] == "train" for record in observations]) + observation_targets = operation_targets(observations) + curriculum_x = np.stack([ + lexical_features(text, 4096) + for operation in OPERATIONS for text in OPERATOR_CURRICULUM[operation] + ]).astype(np.float32) + curriculum_targets = np.zeros((len(curriculum_x), len(OPERATIONS)), dtype=np.float32) + curriculum_row = 0 + for operation in OPERATIONS: + for _ in OPERATOR_CURRICULUM[operation]: + curriculum_targets[curriculum_row, OP_TO_IDX[operation]] = 1.0 + curriculum_row += 1 + + operator_trials = [] + operator_states = {} + for mode in OPERATOR_MODES: + observation_x = instruction_mode_features(observations, archive, mode) + expanded_x = expand_instruction_features(records, observations, observation_x) + train_x = observation_x[observation_train] + train_targets = observation_targets[observation_train] + if mode == "instruction_lexical_curriculum": + train_x = np.concatenate([train_x, curriculum_x], axis=0) + train_targets = np.concatenate([train_targets, curriculum_targets], axis=0) + for ridge in RIDGES: + weights = adaptive_ridge( + train_x, train_targets, ridge, + ) + scores = expanded_x @ weights + trial = { + "mode": mode, + "ridge": ridge, + "feature_dimension": observation_x.shape[1], + "train": operation_metrics(records, scores, "train"), + "dev": operation_metrics(records, scores, "dev"), + } + operator_trials.append(trial) + operator_states[(mode, ridge)] = weights + operator_trials.sort(key=lambda trial: ( + trial["dev"]["macro_f1"], trial["dev"]["minimum_operation_recall"], + -trial["feature_dimension"], trial["ridge"], + ), reverse=True) + selected_operator = operator_trials[0] + + role_x, role_y, role_index = role_pair_dataset(records, split="train") + all_role_x, _, all_role_index = role_pair_dataset(records) + role_trials = [] + role_states = {} + for ridge in RIDGES: + weights = adaptive_ridge(role_x, role_y, ridge) + flat_scores = (all_role_x @ weights).reshape(-1) + score_matrix = np.full((len(records), 4), -np.inf, dtype=np.float32) + for score, (row, candidate) in zip(flat_scores, all_role_index): + score_matrix[row, candidate] = score + trial = { + "ridge": ridge, + "feature_dimension": role_x.shape[1], + "train": role_metrics_from_scores(records, score_matrix, "train"), + "dev": role_metrics_from_scores(records, score_matrix, "dev"), + } + role_trials.append(trial) + role_states[ridge] = weights + role_trials.sort(key=lambda trial: ( + trial["dev"]["correct_over_exact_role_swap"], + trial["dev"]["four_way_accuracy"], trial["ridge"], + ), reverse=True) + selected_role = role_trials[0] + + gates = { + "operator_macro_f1": selected_operator["dev"]["macro_f1"] >= 0.80, + "minimum_operation_recall": selected_operator["dev"]["minimum_operation_recall"] >= 0.70, + "exact_role_swap_discrimination": selected_role["dev"]["correct_over_exact_role_swap"] >= 0.85, + "four_way_role_selection": selected_role["dev"]["four_way_accuracy"] >= 0.80, + "no_confirmation_evaluated": True, + } + metrics = { + "schema_version": "oracle-stage2d-factorized-semantic-development-v3", + "records": len(records), + "unique_instruction_observations": len(observations), + "confirmation_records": 0, + "selected_operator": selected_operator, + "selected_role_binder": selected_role, + "operator_trials": operator_trials, + "role_trials": role_trials, + "gates": gates, + "all_development_gates_pass": all(gates.values()), + "elapsed_seconds": time.perf_counter() - started, + "claim_boundary": ( + "Development-only factorized semantic routing and candidate-conditioned role binding. " + "No answer, Tree reasoning, or conversational claim." + ), + } + args.metrics.parent.mkdir(parents=True, exist_ok=True) + args.metrics.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8") + with args.out.open("wb") as handle: + np.savez_compressed( + handle, + schema_version=np.array("oracle-stage2d-factorized-semantic-specialist-v3"), + operator_mode=np.array(selected_operator["mode"]), + operator_ridge=np.array(selected_operator["ridge"]), + operator_weights=operator_states[(selected_operator["mode"], selected_operator["ridge"])], + role_ridge=np.array(selected_role["ridge"]), + role_weights=role_states[selected_role["ridge"]], + operations=np.asarray(OPERATIONS), + confirmation_evaluated=np.array(False), + ) + print(json.dumps({ + "operator_mode": selected_operator["mode"], + "operator_dev_macro_f1": selected_operator["dev"]["macro_f1"], + "operator_dev_minimum_recall": selected_operator["dev"]["minimum_operation_recall"], + "role_swap_dev": selected_role["dev"]["correct_over_exact_role_swap"], + "role_four_way_dev": selected_role["dev"]["four_way_accuracy"], + "all_development_gates_pass": metrics["all_development_gates_pass"], + "elapsed_seconds": metrics["elapsed_seconds"], + }, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/root/wander-train/train_stage2d_semantic_specialist.py b/root/wander-train/train_stage2d_semantic_specialist.py new file mode 100644 index 0000000000000000000000000000000000000000..d0d51b4f7b211ce8788a98c5859d07dda8201a15 --- /dev/null +++ b/root/wander-train/train_stage2d_semantic_specialist.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Train the Stage 2D semantic operation and role-binding specialist.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +import time +from collections import Counter +from functools import lru_cache +from pathlib import Path +from typing import Any + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from train_stage2a_role_ladder import ( # noqa: E402 + HRR_DIM, OPERATIONS, OP_TO_IDX, as_real, bind, lexical_features, + normalize, text_hrr, unitary, +) +from train_stage2b_malkuth_decoder import macro_f1, wilson_interval # noqa: E402 +from train_stage2c_direct_tree import ( # noqa: E402 + fit_ridge_primal, load_features, load_records, normalize_rows, project, +) + + +MODES = ("no_proposer", "anchor_hidden", "relational_experience") +MODE_SURFACE = {"no_proposer": 0, "anchor_hidden": 1152, "relational_experience": 1802} +RIDGES = (0.01, 0.05, 0.20, 1.00, 5.00) +FEATURE_DIM = 768 +ROLE_TARGET_DIM = 384 + + +def proposer_raw(archive: dict[str, np.ndarray], mode: str) -> np.ndarray: + if mode == "anchor_hidden": + return np.concatenate([ + archive["anchor_block4"], archive["anchor_block5"], archive["anchor_final"], + ], axis=1) + if mode == "relational_experience": + return np.concatenate([ + archive["relational_trunk_final"], archive["relational_hidden"], + archive["relational_experience"], archive["relational_sephirah_logits"], + ], axis=1) + return np.zeros((len(archive["carrier_pooled"]), 1), dtype=np.float32) + + +def specialist_features( + records: list[dict[str, Any]], archive: dict[str, np.ndarray], mode: str, +) -> np.ndarray: + instruction_lexical = np.stack([ + lexical_features(record["instruction_surface"], 512) for record in records + ]).astype(np.float32) + role_lexical = np.stack([ + lexical_features(record["role_clause"], 512) for record in records + ]).astype(np.float32) + instruction_hrr = np.stack([ + as_real(text_hrr(record["instruction_surface"])) for record in records + ]) + instruction_hrr = project(instruction_hrr, "stage2d:semantic:instruction-hrr", 256) + role_hrr = np.stack([ + as_real(text_hrr(record["role_clause"])) for record in records + ]) + role_hrr = project(role_hrr, "stage2d:semantic:role-clause-hrr", 256) + carrier = normalize_rows(np.concatenate([ + archive["carrier_pooled"], archive["carrier_comp_emb"], + archive["carrier_sephirah_logits"], + ], axis=1)) + carrier = project(carrier, "stage2d:semantic:carrier", 384) + raw_proposer = proposer_raw(archive, mode) + proposer = ( + np.zeros((len(records), 256), dtype=np.float32) + if mode == "no_proposer" + else project(raw_proposer, f"stage2d:semantic:{mode}", 256) + ) + combined = np.concatenate([ + instruction_lexical, role_lexical, instruction_hrr, role_hrr, carrier, proposer, + ], axis=1) + linear = combined @ projection_matrix( + f"stage2d:semantic:combined:{mode}", combined.shape[1], FEATURE_DIM, + ) + # A bounded nonlinear random-feature surface lets the small specialist + # represent interactions between semantic request and role language. + return normalize_rows(np.tanh(np.float32(1.75) * linear).astype(np.float32)) + + +def seed_int(label: str) -> int: + return int.from_bytes(hashlib.blake2b(label.encode("utf-8"), digest_size=8).digest(), "little") + + +@lru_cache(maxsize=64) +def projection_matrix(label: str, input_dim: int, output_dim: int) -> np.ndarray: + rng = np.random.default_rng(seed_int(label)) + matrix = rng.normal( + 0.0, 1.0 / math.sqrt(max(input_dim, 1)), (input_dim, output_dim), + ).astype(np.float32) + return matrix / np.maximum(np.linalg.norm(matrix, axis=0, keepdims=True), 1e-8) + + +def operation_targets(records: list[dict[str, Any]]) -> np.ndarray: + targets = np.zeros((len(records), len(OPERATIONS)), dtype=np.float32) + for row, record in enumerate(records): + targets[row, OP_TO_IDX[record["operation"]]] = 1.0 + return targets + + +def role_map_hrr(role_map: dict[str, str]) -> np.ndarray: + state = np.zeros(HRR_DIM, dtype=np.complex64) + for role in ("source", "field", "transition"): + state += bind( + unitary(f"stage2d:semantic:role:{role}"), text_hrr(role_map[role]), + ) + return normalize(state) + + +def role_candidate_embedding(role_map: dict[str, str]) -> np.ndarray: + serialized = " | ".join(f"{role}={role_map[role]}" for role in ("source", "field", "transition")) + lexical = lexical_features(serialized, 128) + geometric = as_real(role_map_hrr(role_map)) @ projection_matrix( + "stage2d:semantic:role-candidate-hrr", 2 * HRR_DIM, 256, + ) + value = np.concatenate([lexical, geometric]).astype(np.float32) + return value / max(float(np.linalg.norm(value)), 1e-8) + + +def role_candidates(records: list[dict[str, Any]]) -> np.ndarray: + return np.stack([ + np.stack([role_candidate_embedding(candidate["map"]) for candidate in record["role_candidates"]]) + for record in records + ]).astype(np.float32) + + +def role_contrast_targets(candidates: np.ndarray) -> np.ndarray: + correct = candidates[:, 0] + wrong = candidates[:, 1:].mean(axis=1) + return normalize_rows(correct - wrong) + + +def operation_metrics( + records: list[dict[str, Any]], scores: np.ndarray, split: str, +) -> dict[str, Any]: + mask = np.asarray([record["split"] == split for record in records]) + truth = np.asarray([OP_TO_IDX[record["operation"]] for record in records])[mask] + predicted = scores[mask].argmax(axis=1) + confusion = np.zeros((len(OPERATIONS), len(OPERATIONS)), dtype=int) + for actual, guess in zip(truth, predicted): + confusion[actual, guess] += 1 + recalls = {} + for index, operation in enumerate(OPERATIONS): + actual = truth == index + recalls[operation] = float(np.mean(predicted[actual] == index)) + by_depth = {} + split_rows = np.flatnonzero(mask) + for depth in (0, 2, 4, 8): + local = np.asarray([ + records[row]["distractor_turn_count"] == depth for row in split_rows + ]) + by_depth[str(depth)] = float(np.mean(predicted[local] == truth[local])) + correct = int(np.sum(predicted == truth)) + return { + "records": len(truth), + "correct": correct, + "accuracy": correct / len(truth), + "accuracy_wilson_95": wilson_interval(len(truth), correct), + "macro_f1": macro_f1(truth, predicted, len(OPERATIONS)), + "minimum_operation_recall": min(recalls.values()), + "recall_by_operation": recalls, + "accuracy_by_distractor_turns": by_depth, + "confusion": confusion.tolist(), + "predicted_counts": dict(Counter(OPERATIONS[index] for index in predicted)), + } + + +def role_metrics( + records: list[dict[str, Any]], predicted: np.ndarray, + candidates: np.ndarray, split: str, +) -> dict[str, Any]: + mask = np.asarray([record["split"] == split for record in records]) + scores = np.einsum("nd,nkd->nk", predicted[mask], candidates[mask]) + choices = scores.argmax(axis=1) + correct = int(np.sum(choices == 0)) + swap_wins = scores[:, 0] > scores[:, 1] + by_depth = {} + split_rows = np.flatnonzero(mask) + for depth in (0, 2, 4, 8): + local = np.asarray([ + records[row]["distractor_turn_count"] == depth for row in split_rows + ]) + by_depth[str(depth)] = { + "four_way_accuracy": float(np.mean(choices[local] == 0)), + "correct_over_exact_swap": float(np.mean(swap_wins[local])), + } + return { + "records": len(choices), + "correct": correct, + "four_way_accuracy": correct / len(choices), + "correct_over_exact_role_swap": float(np.mean(swap_wins)), + "choice_counts": dict(Counter( + records[row]["role_candidates"][choice]["kind"] + for row, choice in zip(np.flatnonzero(mask), choices) + )), + "by_distractor_turns": by_depth, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--corpus", required=True, type=Path) + parser.add_argument("--features", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--metrics", required=True, type=Path) + args = parser.parse_args() + started = time.perf_counter() + records = load_records(args.corpus) + if {record["split"] for record in records} != {"train", "dev"}: + raise ValueError("Stage 2D semantic development accepts train/dev only") + archive = load_features(records, args.features) + for name, value in archive.items(): + if isinstance(value, np.ndarray) and not np.isfinite(value).all(): + raise FloatingPointError(f"non-finite frozen feature: {name}") + train_mask = np.asarray([record["split"] == "train" for record in records]) + op_targets = operation_targets(records) + candidates = role_candidates(records) + role_targets = role_contrast_targets(candidates) + joint_targets = np.concatenate([op_targets, role_targets], axis=1) + trials = [] + states: dict[tuple[str, float], dict[str, np.ndarray]] = {} + for mode in MODES: + features = specialist_features(records, archive, mode) + for ridge in RIDGES: + joint_weights = fit_ridge_primal( + features[train_mask], joint_targets[train_mask], ridge, + ) + op_weights = joint_weights[:, :len(OPERATIONS)] + role_weights = joint_weights[:, len(OPERATIONS):] + op_scores = features @ op_weights + role_prediction = normalize_rows(features @ role_weights) + trial = { + "mode": mode, + "ridge": ridge, + "trainable_input_surface": MODE_SURFACE[mode], + "operator": { + "train": operation_metrics(records, op_scores, "train"), + "dev": operation_metrics(records, op_scores, "dev"), + }, + "roles": { + "train": role_metrics(records, role_prediction, candidates, "train"), + "dev": role_metrics(records, role_prediction, candidates, "dev"), + }, + } + trials.append(trial) + states[(mode, ridge)] = { + "operator_weights": op_weights, + "role_weights": role_weights, + } + trials.sort(key=lambda trial: ( + trial["operator"]["dev"]["macro_f1"], + trial["operator"]["dev"]["minimum_operation_recall"], + trial["roles"]["dev"]["correct_over_exact_role_swap"], + trial["roles"]["dev"]["four_way_accuracy"], + -trial["trainable_input_surface"], + trial["ridge"], + ), reverse=True) + selected = trials[0] + op_dev = selected["operator"]["dev"] + role_dev = selected["roles"]["dev"] + gates = { + "operator_macro_f1": op_dev["macro_f1"] >= 0.80, + "minimum_operation_recall": op_dev["minimum_operation_recall"] >= 0.70, + "exact_role_swap_discrimination": role_dev["correct_over_exact_role_swap"] >= 0.85, + "four_way_role_selection": role_dev["four_way_accuracy"] >= 0.80, + "no_confirmation_evaluated": True, + } + metrics = { + "schema_version": "oracle-stage2d-semantic-specialist-development-v1", + "records": len(records), + "train_records": int(train_mask.sum()), + "dev_records": int((~train_mask).sum()), + "confirmation_records": 0, + "selection_rule": ( + "dev operator macro-F1, minimum recall, role-swap discrimination, " + "four-way role accuracy, smaller proposer surface, larger ridge" + ), + "selected": selected, + "trials": trials, + "gates": gates, + "all_development_gates_pass": all(gates.values()), + "elapsed_seconds": time.perf_counter() - started, + "claim_boundary": ( + "Development-only semantic and role specialization. No Stage 2D " + "confirmation set exists, and no prospective generalization claim is made." + ), + } + args.metrics.parent.mkdir(parents=True, exist_ok=True) + args.metrics.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8") + chosen = states[(selected["mode"], selected["ridge"])] + with args.out.open("wb") as handle: + np.savez_compressed( + handle, + schema_version=np.array("oracle-stage2d-semantic-specialist-checkpoint-v1"), + selected_mode=np.array(selected["mode"]), + selected_ridge=np.array(selected["ridge"]), + operator_weights=chosen["operator_weights"], + role_weights=chosen["role_weights"], + operations=np.asarray(OPERATIONS), + confirmation_evaluated=np.array(False), + ) + print(json.dumps({ + "selected_mode": selected["mode"], + "selected_ridge": selected["ridge"], + "operator_dev_macro_f1": op_dev["macro_f1"], + "operator_dev_minimum_recall": op_dev["minimum_operation_recall"], + "role_swap_dev": role_dev["correct_over_exact_role_swap"], + "role_four_way_dev": role_dev["four_way_accuracy"], + "all_development_gates_pass": metrics["all_development_gates_pass"], + "elapsed_seconds": metrics["elapsed_seconds"], + }, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())