File size: 2,555 Bytes
dca70d8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """Load supermix-v93 and ask it a question.
python example_usage.py "What is 47 x 6?"
python example_usage.py "Trace this Python and give x. x = 5
for i in range(4): x = x + 7"
The model answers in the format it was trained on, so the question is
normalised first -- see the model card. `prompt_normaliser` prints what it
actually sent, and `answer_check` independently re-derives the answer, so a
wrong reply is reported as wrong rather than presented as fact. For a code
question `answer_check` re-derives by **running the snippet**.
`step_audit` then reads the reply's own working and reports the first step that
disagrees with exact arithmetic. That matters here: v87's `power` task writes a
confident, well-formed derivation whose middle steps are false, and the answer
alone does not show you that.
"""
from __future__ import annotations
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE / "src"))
import answer_check # noqa: E402
import prompt_normaliser # noqa: E402
import step_audit # noqa: E402
from train_mimomix_talk import generate_reply, load_talk_checkpoint # noqa: E402
DEFAULT = "Trace this Python and give x. x = 5\nfor i in range(4): x = x + 7"
def main() -> int:
question = " ".join(sys.argv[1:]) or DEFAULT
model, tokenizer, _ = load_talk_checkpoint(str(HERE / "supermix_v93.pt"))
model.eval()
rewritten = prompt_normaliser.normalise(question)
if rewritten.changed:
print(f"asked as : {rewritten.prompt} ({rewritten.rule})")
result = generate_reply(model, tokenizer, rewritten.prompt, max_new_tokens=96)
reply = result["reply"] if isinstance(result, dict) else str(result)
print(f"reply : {reply}")
verdict = answer_check.check(rewritten.prompt, reply)
if verdict is None:
print("check : NOT CHECKED (not a question this can verify)")
elif verdict.correct:
print(f"check : CORRECT ({verdict.expected})")
else:
print(f"check : WRONG (answered {verdict.predicted}, "
f"expected {verdict.expected})")
audit = step_audit.audit(reply)
bad = audit.first_bad
if bad is not None:
print(f"working : step {bad.position} is false -- '{bad.text}' "
f"states {bad.stated:g}, arithmetic gives {bad.expected:g}")
elif audit.written:
print(f"working : all {len(audit.written)} written steps check out "
f"({audit.verdict})")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|