File size: 1,847 Bytes
15e7062 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | """Next Trace inference (NumPy only): evidence sentences or abstention for a question over a document.
Usage:
python inference.py --selftest
python inference.py --doc "..." --question "..."
"""
import argparse
import json
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import trace_core as tc # noqa: E402
from nlrt import load_model # noqa: E402
def load(path=os.path.join(HERE, "model.json")):
return load_model(path)
def trace(model, question, document):
r = tc.score_document(model, question, document)
return {"abstain": r["abstain"], "answerable": r["answerable"],
"evidence": [{"index": i, "sentence": document[r["spans"][i][0]:r["spans"][i][1]], "p": r["probs"][i]} for i in r["evidence"]]}
def _selftest():
model = load()
with open(os.path.join(HERE, "selftest.json"), encoding="utf-8") as f:
cases = json.load(f)["cases"]
worst, bad = 0.0, []
for c in cases:
r = tc.score_document(model, c["question"], c["document"])
worst = max([worst] + [abs(a - b) for a, b in zip(r["probs"], c["probs"])] + [abs(r["answerable"] - c["answerable"])])
if r["abstain"] != c["abstain"] or r["evidence"] != c["evidence"]:
bad.append(c["question"])
ok = not bad and worst < 1e-6
print(json.dumps({"passed": ok, "cases": len(cases), "max_abs_diff": worst, "decision_mismatches": bad}, ensure_ascii=False))
return ok
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--selftest", action="store_true")
ap.add_argument("--doc", default="")
ap.add_argument("--question", default="")
a = ap.parse_args()
if a.selftest:
sys.exit(0 if _selftest() else 1)
print(json.dumps(trace(load(), a.question, a.doc), ensure_ascii=False, indent=1))
|