Instructions to use wordspace/decider-1.7B-MLX-6bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use wordspace/decider-1.7B-MLX-6bit with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download wordspace/decider-1.7B-MLX-6bit --local-dir decider-1.7B-MLX-6bit
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download example.py from wordspace/decider-1.7B-MLX-6bit: direct link, hf CLI and curl.
- Browser
- Download file 2.44 kB
-
https://huggingface.co/wordspace/decider-1.7B-MLX-6bit/resolve/main/example.py
- Command line
-
hf download hf://wordspace/decider-1.7B-MLX-6bit/example.py
-
curl -L -o example.py https://huggingface.co/wordspace/decider-1.7B-MLX-6bit/resolve/main/example.py
2.44 kB
| """One decision per question: a single forward pass, the option scores, and a | |
| calibrated probability. Run it from the model folder: | |
| python example.py | |
| Needs `mlx-lm` (Apple silicon). The question wording lives in `protocol.py` | |
| and must not be changed: the model was trained on exactly that text. | |
| """ | |
| import json | |
| import math | |
| import os | |
| import mlx.core as mx | |
| from mlx_lm import load | |
| import protocol as P | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| def option_tokens(tokenizer, question): | |
| """Token ids for each option: the forms in `P.variants` that are a single token.""" | |
| out = [] | |
| for forms in P.variants(question): | |
| ids = [] | |
| for f in forms: | |
| e = tokenizer.encode(f, add_special_tokens=False) | |
| if len(e) == 1 and e[0] not in ids: | |
| ids.append(e[0]) | |
| out.append(ids) | |
| return out | |
| def decide(model, tokenizer, state, question, temperature): | |
| """Probabilities of the options, in the order of `P.answers(question)`.""" | |
| ids = tokenizer.encode(P.prompt(state, question), add_special_tokens=False) | |
| logits = model(mx.array(ids)[None])[0, -1].astype(mx.float32) | |
| logp = (logits - mx.logsumexp(logits, axis=-1, keepdims=True)).tolist() | |
| scores = [max(logp[i] for i in group) for group in option_tokens(tokenizer, question)] | |
| m = max(scores) | |
| e = [math.exp((s - m) / temperature) for s in scores] | |
| return [x / sum(e) for x in e] | |
| def main(): | |
| model, tokenizer = load(HERE) | |
| with open(os.path.join(HERE, "calibration.json")) as f: | |
| t = json.load(f)["temperature"] | |
| cases = [ | |
| ("contradiction", P.pair_state( | |
| "a0000001-0000-4000-8000-000000000001", "The dentist appointment is on Tuesday at 10.", | |
| "a0000001-0000-4000-8000-000000000002", "Reminder: dentist on Thursday at 10.")), | |
| ("hypothesis", "Marco suggested moving the dinner to Friday."), | |
| ("update", P.update_state( | |
| "The rent is 900 euros a month.", "From March the rent goes up to 950 euros.")), | |
| ("membership", P.membership_state( | |
| "Garden", "Plants, sowing and watering on the balcony.", | |
| "Repotted the basil and moved the tomatoes into the sun.")), | |
| ] | |
| for question, state in cases: | |
| p = decide(model, tokenizer, state, question, t) | |
| print(question, " ".join("%s %.3f" % (a, x) for a, x in zip(P.answers(question), p))) | |
| if __name__ == "__main__": | |
| main() | |