File size: 1,847 Bytes
15e7062
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
"""Next Trace inference (NumPy only): evidence sentences or abstention for a question over a document.

Usage:
    python inference.py --selftest
    python inference.py --doc "..." --question "..."
"""
import argparse
import json
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)

import trace_core as tc  # noqa: E402
from nlrt import load_model  # noqa: E402


def load(path=os.path.join(HERE, "model.json")):
    return load_model(path)


def trace(model, question, document):
    r = tc.score_document(model, question, document)
    return {"abstain": r["abstain"], "answerable": r["answerable"],
            "evidence": [{"index": i, "sentence": document[r["spans"][i][0]:r["spans"][i][1]], "p": r["probs"][i]} for i in r["evidence"]]}


def _selftest():
    model = load()
    with open(os.path.join(HERE, "selftest.json"), encoding="utf-8") as f:
        cases = json.load(f)["cases"]
    worst, bad = 0.0, []
    for c in cases:
        r = tc.score_document(model, c["question"], c["document"])
        worst = max([worst] + [abs(a - b) for a, b in zip(r["probs"], c["probs"])] + [abs(r["answerable"] - c["answerable"])])
        if r["abstain"] != c["abstain"] or r["evidence"] != c["evidence"]:
            bad.append(c["question"])
    ok = not bad and worst < 1e-6
    print(json.dumps({"passed": ok, "cases": len(cases), "max_abs_diff": worst, "decision_mismatches": bad}, ensure_ascii=False))
    return ok


if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--selftest", action="store_true")
    ap.add_argument("--doc", default="")
    ap.add_argument("--question", default="")
    a = ap.parse_args()
    if a.selftest:
        sys.exit(0 if _selftest() else 1)
    print(json.dumps(trace(load(), a.question, a.doc), ensure_ascii=False, indent=1))