Next-Trace / inference.py
Lamapi's picture
Publish Next-Trace model
15e7062 verified
Raw History Blame Contribute Delete
1.85 kB
"""Next Trace inference (NumPy only): evidence sentences or abstention for a question over a document.
Usage:
python inference.py --selftest
python inference.py --doc "..." --question "..."
"""
import argparse
import json
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import trace_core as tc # noqa: E402
from nlrt import load_model # noqa: E402
def load(path=os.path.join(HERE, "model.json")):
return load_model(path)
def trace(model, question, document):
r = tc.score_document(model, question, document)
return {"abstain": r["abstain"], "answerable": r["answerable"],
"evidence": [{"index": i, "sentence": document[r["spans"][i][0]:r["spans"][i][1]], "p": r["probs"][i]} for i in r["evidence"]]}
def _selftest():
model = load()
with open(os.path.join(HERE, "selftest.json"), encoding="utf-8") as f:
cases = json.load(f)["cases"]
worst, bad = 0.0, []
for c in cases:
r = tc.score_document(model, c["question"], c["document"])
worst = max([worst] + [abs(a - b) for a, b in zip(r["probs"], c["probs"])] + [abs(r["answerable"] - c["answerable"])])
if r["abstain"] != c["abstain"] or r["evidence"] != c["evidence"]:
bad.append(c["question"])
ok = not bad and worst < 1e-6
print(json.dumps({"passed": ok, "cases": len(cases), "max_abs_diff": worst, "decision_mismatches": bad}, ensure_ascii=False))
return ok
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--selftest", action="store_true")
ap.add_argument("--doc", default="")
ap.add_argument("--question", default="")
a = ap.parse_args()
if a.selftest:
sys.exit(0 if _selftest() else 1)
print(json.dumps(trace(load(), a.question, a.doc), ensure_ascii=False, indent=1))