decider-1.7B-MLX-6bit / example.py
dmottad's picture
English release: decider, protocol.py, example.py, calibration.json; model card for the LLM wiki
1825c16 verified
Raw History Blame Contribute Delete
2.44 kB
"""One decision per question: a single forward pass, the option scores, and a
calibrated probability. Run it from the model folder:
python example.py
Needs `mlx-lm` (Apple silicon). The question wording lives in `protocol.py`
and must not be changed: the model was trained on exactly that text.
"""
import json
import math
import os
import mlx.core as mx
from mlx_lm import load
import protocol as P
HERE = os.path.dirname(os.path.abspath(__file__))
def option_tokens(tokenizer, question):
"""Token ids for each option: the forms in `P.variants` that are a single token."""
out = []
for forms in P.variants(question):
ids = []
for f in forms:
e = tokenizer.encode(f, add_special_tokens=False)
if len(e) == 1 and e[0] not in ids:
ids.append(e[0])
out.append(ids)
return out
def decide(model, tokenizer, state, question, temperature):
"""Probabilities of the options, in the order of `P.answers(question)`."""
ids = tokenizer.encode(P.prompt(state, question), add_special_tokens=False)
logits = model(mx.array(ids)[None])[0, -1].astype(mx.float32)
logp = (logits - mx.logsumexp(logits, axis=-1, keepdims=True)).tolist()
scores = [max(logp[i] for i in group) for group in option_tokens(tokenizer, question)]
m = max(scores)
e = [math.exp((s - m) / temperature) for s in scores]
return [x / sum(e) for x in e]
def main():
model, tokenizer = load(HERE)
with open(os.path.join(HERE, "calibration.json")) as f:
t = json.load(f)["temperature"]
cases = [
("contradiction", P.pair_state(
"a0000001-0000-4000-8000-000000000001", "The dentist appointment is on Tuesday at 10.",
"a0000001-0000-4000-8000-000000000002", "Reminder: dentist on Thursday at 10.")),
("hypothesis", "Marco suggested moving the dinner to Friday."),
("update", P.update_state(
"The rent is 900 euros a month.", "From March the rent goes up to 950 euros.")),
("membership", P.membership_state(
"Garden", "Plants, sowing and watering on the balcony.",
"Repotted the basil and moved the tomatoes into the sun.")),
]
for question, state in cases:
p = decide(model, tokenizer, state, question, t)
print(question, " ".join("%s %.3f" % (a, x) for a, x in zip(P.answers(question), p)))
if __name__ == "__main__":
main()