Download wiki_model/cli.py from Lucasgitguy-co/wiki-model-tiny: direct link, hf CLI and curl.
- Browser
- Download file 9.64 kB
-
https://huggingface.co/Lucasgitguy-co/wiki-model-tiny/resolve/main/wiki_model/cli.py
- Command line
-
hf download hf://Lucasgitguy-co/wiki-model-tiny/wiki_model/cli.py
-
curl -L -o cli.py https://huggingface.co/Lucasgitguy-co/wiki-model-tiny/resolve/main/wiki_model/cli.py
9.64 kB
| """wiki-model command line. | |
| wiki-model "Eiffel Tower" a fact about a topic | |
| wiki-model --random a random fact | |
| wiki-model --q "Who wrote Hamlet?" question answering | |
| wiki-model --fix "The Moon is made of cheese." correct a false statement | |
| add --grounded to answer from the sourced Wikipedia index (shows the source URL) | |
| --model tiny|135m (default: tiny) | |
| """ | |
| import argparse, json, os, random, re, sys | |
| from pathlib import Path | |
| from .prompts import fact, question, fix, CANT | |
| HERE = Path(__file__).resolve().parent | |
| ROOT = HERE.parent | |
| def _device(want): | |
| import torch | |
| if want != "auto": return want | |
| return "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu" | |
| class Tiny: | |
| name = "wiki-model-tiny" | |
| def __init__(self, path, device="cpu"): | |
| from tokenizers import Tokenizer | |
| from .model import TinyGPT | |
| self.m = TinyGPT.load(path, device) | |
| tj = Path(path) / "tokenizer.json" | |
| self.tok = Tokenizer.from_file(str(tj if tj.exists() else HERE / "tok" / "tokenizer.json")) | |
| self.eos = self.tok.token_to_id("<eos>") | |
| def __call__(self, prompt, max_new=80, temperature=0.0, seed=None, stop=None): | |
| import torch | |
| g = torch.Generator().manual_seed(seed) if seed is not None else None | |
| ids = self.tok.encode(prompt).ids | |
| out = self.tok.decode(self.m.generate(ids, max_new, self.eos, temperature, gen=g), skip_special_tokens=False) | |
| if stop and stop in out: out = out.split(stop)[0] | |
| return out.replace("<eos>", "").strip() | |
| class _AnswerNoRepeat: | |
| """Block 4-gram repeats inside the generated answer only. (transformers' no_repeat_ngram_size also | |
| counts the prompt, which forbids copying a claim or a value from <ctx>.)""" | |
| def __init__(self, prompt_len, n=4): | |
| self.p, self.n = prompt_len, n | |
| def __call__(self, input_ids, scores): | |
| for b, seq in enumerate(input_ids[:, self.p:].tolist()): | |
| if len(seq) < self.n - 1: continue | |
| tail = tuple(seq[-(self.n - 1):]) | |
| banned = [seq[i + self.n - 1] for i in range(len(seq) - self.n + 1) if tuple(seq[i:i + self.n - 1]) == tail] | |
| if banned: scores[b, banned] = float("-inf") | |
| return scores | |
| class Smol: | |
| name = "wiki-model" | |
| def __init__(self, path, device="cpu"): | |
| import logging | |
| logging.getLogger("torch.distributed.elastic.multiprocessing.redirects").setLevel(logging.ERROR) | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| from transformers.utils import logging as tlog | |
| tlog.set_verbosity_error(); tlog.disable_progress_bar() | |
| self.tok = AutoTokenizer.from_pretrained(path) | |
| try: | |
| m = AutoModelForCausalLM.from_pretrained(path, dtype=torch.float32) | |
| except TypeError: # transformers < 4.56 only knows torch_dtype | |
| m = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.float32) | |
| self.m = m.to(device).eval() | |
| self.dev = device | |
| def __call__(self, prompt, max_new=80, temperature=0.0, seed=None, stop=None): | |
| import torch | |
| if seed is not None: torch.manual_seed(seed) | |
| x = self.tok(prompt, return_tensors="pt", add_special_tokens=False).to(self.dev) | |
| kw = dict(do_sample=True, temperature=temperature, top_p=0.9) if temperature > 0 else dict(do_sample=False) | |
| with torch.no_grad(): | |
| y = self.m.generate(**x, max_new_tokens=max_new, eos_token_id=self.tok.eos_token_id, | |
| logits_processor=[_AnswerNoRepeat(x["input_ids"].shape[1])], | |
| pad_token_id=self.tok.eos_token_id, **kw) | |
| out = self.tok.decode(y[0, x["input_ids"].shape[1]:], skip_special_tokens=True) | |
| if stop and stop in out: out = out.split(stop)[0] | |
| return out.strip() | |
| def default_model_path(kind): | |
| env = os.environ.get("WIKI_MODEL_TINY" if kind == "tiny" else "WIKI_MODEL_135M") | |
| if env: return env | |
| cfg = ROOT / "config.json" # running from inside a downloaded HF repo (model files next to wiki_model/) | |
| if cfg.exists() and (ROOT / "model.safetensors").exists(): | |
| import json | |
| if ("model_type" in json.load(open(cfg))) == (kind == "135m"): return str(ROOT) | |
| local = ROOT / "runs" / ("tiny-sft" if kind == "tiny" else "smol-sft") / "final" | |
| if local.exists(): return str(local) | |
| from huggingface_hub import snapshot_download | |
| return snapshot_download("Lucasgitguy-co/" + ("wiki-model-tiny" if kind == "tiny" else "wiki-model")) | |
| def default_index_path(): | |
| env = os.environ.get("WIKI_MODEL_INDEX") | |
| if env: return env | |
| local = ROOT / "data" / "index" | |
| if (local / "docs.sqlite").exists(): return str(local) | |
| from huggingface_hub import snapshot_download | |
| return str(Path(snapshot_download("Lucasgitguy-co/wiki-model-facts", repo_type="dataset", | |
| allow_patterns=["index/*", "index/bm25/*"])) / "index") | |
| def load_model(kind, path=None, device="auto"): | |
| dev = _device(device) | |
| return (Tiny if kind == "tiny" else Smol)(path or default_model_path(kind), dev) | |
| def context_for(index, query, topic=None): | |
| """Same shape as training <ctx>: best article text, optionally ' | ' + a shorter second hit.""" | |
| hits = [] | |
| if topic and (d := index.by_title(topic)): hits.append(d) | |
| for h in index.search(query, 3): | |
| if all(h["title"] != x["title"] for x in hits): hits.append(h) | |
| if not hits: return None, [] | |
| ctx = hits[0]["text"] | |
| if len(hits) > 1: ctx += " | " + hits[1]["text"][:250] | |
| return ctx, hits[:2] | |
| def run(model, task, text, index=None, grounded=False, temperature=0.0, seed=None): | |
| """Returns dict(answer, sources, support). task in fact|q|fix|random.""" | |
| rng = random.Random(seed) | |
| if task == "random": # a real topic (index, or the bundled list of training topics); the model states the fact | |
| if index is not None: | |
| text = index.random(rng)["title"] | |
| else: | |
| text = rng.choice((HERE / "topics.txt").read_text().split("\n")[:-1]) | |
| task = "fact" | |
| ctx, hits = (context_for(index, text, topic=text if task == "fact" else None) if grounded and index else (None, [])) | |
| prompt = {"fact": fact, "q": question, "fix": fix}[task](text, ctx) | |
| ans = model(prompt, max_new=100, temperature=temperature, seed=seed) | |
| ans = re.sub(r"\s+", " ", ans).strip() | |
| res = {"task": task, "input": text, "answer": ans, "sources": [{"title": h["title"], "url": h["url"]} for h in hits]} | |
| if index is not None and not grounded and task in ("fact", "fix") and ans and ans != CANT: | |
| f1, src = index.support(ans, topic=text if task == "fact" else None) | |
| res["support"] = {"f1": round(f1, 2), "title": src and src["title"], "url": src and src["url"], | |
| "label": "supported" if f1 >= 0.5 else "unverified"} | |
| return res | |
| def main(argv=None): | |
| ap = argparse.ArgumentParser(prog="wiki-model", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("topic", nargs="*", help="topic to state a fact about") | |
| ap.add_argument("--q", help="ask a question") | |
| ap.add_argument("--fix", help="correct a (possibly false) statement") | |
| ap.add_argument("--random", action="store_true", help="a random fact") | |
| ap.add_argument("--grounded", action="store_true", help="answer from the sourced Wikipedia index") | |
| ap.add_argument("--model", choices=["tiny", "135m"], default=None, | |
| help="default: the weights next to this package (a downloaded repo), else tiny") | |
| ap.add_argument("--model-path"); ap.add_argument("--index") | |
| ap.add_argument("--no-index", action="store_true", help="skip the index (no support check)") | |
| ap.add_argument("--n", type=int, default=1, help="number of answers (sampled when > 1)") | |
| ap.add_argument("--temperature", type=float, default=None) | |
| ap.add_argument("--seed", type=int); ap.add_argument("--device", default="auto") | |
| ap.add_argument("--json", action="store_true") | |
| a = ap.parse_args(argv) | |
| if a.q: task, text = "q", a.q | |
| elif a.fix: task, text = "fix", a.fix | |
| elif a.random: task, text = "random", None | |
| elif a.topic: task, text = "fact", " ".join(a.topic) | |
| else: ap.print_help(); return 1 | |
| if a.model is None: | |
| cfg = ROOT / "config.json" | |
| a.model = "135m" if cfg.exists() and "model_type" in json.loads(cfg.read_text()) else "tiny" | |
| model = load_model(a.model, a.model_path, a.device) | |
| index = None | |
| if a.grounded or not a.no_index: | |
| try: | |
| from .index import FactIndex | |
| index = FactIndex(a.index or default_index_path()) | |
| except Exception as e: | |
| if a.grounded: sys.exit(f"--grounded needs the fact index ({e})") | |
| temp = a.temperature if a.temperature is not None else (0.8 if a.n > 1 or task == "random" else 0.0) | |
| for i in range(a.n): | |
| seed = (a.seed if a.seed is not None else random.randrange(1 << 30)) + i | |
| r = run(model, task, text, index, a.grounded, temp, seed) | |
| if a.json: print(json.dumps(r, ensure_ascii=False)); continue | |
| if task == "random": print(f"[{r['input']}]") | |
| print(r["answer"]) | |
| for s in r["sources"][:1]: print(f" source: {s['title']} — {s['url']}") | |
| if "support" in r: | |
| s = r["support"] | |
| print(f" {s['label']} (overlap {s['f1']})" + (f" — closest: {s['url']}" if s["url"] else "")) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |