"""wiki-model command line. wiki-model "Eiffel Tower" a fact about a topic wiki-model --random a random fact wiki-model --q "Who wrote Hamlet?" question answering wiki-model --fix "The Moon is made of cheese." correct a false statement add --grounded to answer from the sourced Wikipedia index (shows the source URL) --model tiny|135m (default: tiny) """ import argparse, json, os, random, re, sys from pathlib import Path from .prompts import fact, question, fix, CANT HERE = Path(__file__).resolve().parent ROOT = HERE.parent def _device(want): import torch if want != "auto": return want return "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu" class Tiny: name = "wiki-model-tiny" def __init__(self, path, device="cpu"): from tokenizers import Tokenizer from .model import TinyGPT self.m = TinyGPT.load(path, device) tj = Path(path) / "tokenizer.json" self.tok = Tokenizer.from_file(str(tj if tj.exists() else HERE / "tok" / "tokenizer.json")) self.eos = self.tok.token_to_id("") def __call__(self, prompt, max_new=80, temperature=0.0, seed=None, stop=None): import torch g = torch.Generator().manual_seed(seed) if seed is not None else None ids = self.tok.encode(prompt).ids out = self.tok.decode(self.m.generate(ids, max_new, self.eos, temperature, gen=g), skip_special_tokens=False) if stop and stop in out: out = out.split(stop)[0] return out.replace("", "").strip() class _AnswerNoRepeat: """Block 4-gram repeats inside the generated answer only. (transformers' no_repeat_ngram_size also counts the prompt, which forbids copying a claim or a value from .)""" def __init__(self, prompt_len, n=4): self.p, self.n = prompt_len, n def __call__(self, input_ids, scores): for b, seq in enumerate(input_ids[:, self.p:].tolist()): if len(seq) < self.n - 1: continue tail = tuple(seq[-(self.n - 1):]) banned = [seq[i + self.n - 1] for i in range(len(seq) - self.n + 1) if tuple(seq[i:i + self.n - 1]) == tail] if banned: scores[b, banned] = float("-inf") return scores class Smol: name = "wiki-model" def __init__(self, path, device="cpu"): import logging logging.getLogger("torch.distributed.elastic.multiprocessing.redirects").setLevel(logging.ERROR) import torch from transformers import AutoModelForCausalLM, AutoTokenizer from transformers.utils import logging as tlog tlog.set_verbosity_error(); tlog.disable_progress_bar() self.tok = AutoTokenizer.from_pretrained(path) try: m = AutoModelForCausalLM.from_pretrained(path, dtype=torch.float32) except TypeError: # transformers < 4.56 only knows torch_dtype m = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.float32) self.m = m.to(device).eval() self.dev = device def __call__(self, prompt, max_new=80, temperature=0.0, seed=None, stop=None): import torch if seed is not None: torch.manual_seed(seed) x = self.tok(prompt, return_tensors="pt", add_special_tokens=False).to(self.dev) kw = dict(do_sample=True, temperature=temperature, top_p=0.9) if temperature > 0 else dict(do_sample=False) with torch.no_grad(): y = self.m.generate(**x, max_new_tokens=max_new, eos_token_id=self.tok.eos_token_id, logits_processor=[_AnswerNoRepeat(x["input_ids"].shape[1])], pad_token_id=self.tok.eos_token_id, **kw) out = self.tok.decode(y[0, x["input_ids"].shape[1]:], skip_special_tokens=True) if stop and stop in out: out = out.split(stop)[0] return out.strip() def default_model_path(kind): env = os.environ.get("WIKI_MODEL_TINY" if kind == "tiny" else "WIKI_MODEL_135M") if env: return env cfg = ROOT / "config.json" # running from inside a downloaded HF repo (model files next to wiki_model/) if cfg.exists() and (ROOT / "model.safetensors").exists(): import json if ("model_type" in json.load(open(cfg))) == (kind == "135m"): return str(ROOT) local = ROOT / "runs" / ("tiny-sft" if kind == "tiny" else "smol-sft") / "final" if local.exists(): return str(local) from huggingface_hub import snapshot_download return snapshot_download("Lucasgitguy-co/" + ("wiki-model-tiny" if kind == "tiny" else "wiki-model")) def default_index_path(): env = os.environ.get("WIKI_MODEL_INDEX") if env: return env local = ROOT / "data" / "index" if (local / "docs.sqlite").exists(): return str(local) from huggingface_hub import snapshot_download return str(Path(snapshot_download("Lucasgitguy-co/wiki-model-facts", repo_type="dataset", allow_patterns=["index/*", "index/bm25/*"])) / "index") def load_model(kind, path=None, device="auto"): dev = _device(device) return (Tiny if kind == "tiny" else Smol)(path or default_model_path(kind), dev) def context_for(index, query, topic=None): """Same shape as training : best article text, optionally ' | ' + a shorter second hit.""" hits = [] if topic and (d := index.by_title(topic)): hits.append(d) for h in index.search(query, 3): if all(h["title"] != x["title"] for x in hits): hits.append(h) if not hits: return None, [] ctx = hits[0]["text"] if len(hits) > 1: ctx += " | " + hits[1]["text"][:250] return ctx, hits[:2] def run(model, task, text, index=None, grounded=False, temperature=0.0, seed=None): """Returns dict(answer, sources, support). task in fact|q|fix|random.""" rng = random.Random(seed) if task == "random": # a real topic (index, or the bundled list of training topics); the model states the fact if index is not None: text = index.random(rng)["title"] else: text = rng.choice((HERE / "topics.txt").read_text().split("\n")[:-1]) task = "fact" ctx, hits = (context_for(index, text, topic=text if task == "fact" else None) if grounded and index else (None, [])) prompt = {"fact": fact, "q": question, "fix": fix}[task](text, ctx) ans = model(prompt, max_new=100, temperature=temperature, seed=seed) ans = re.sub(r"\s+", " ", ans).strip() res = {"task": task, "input": text, "answer": ans, "sources": [{"title": h["title"], "url": h["url"]} for h in hits]} if index is not None and not grounded and task in ("fact", "fix") and ans and ans != CANT: f1, src = index.support(ans, topic=text if task == "fact" else None) res["support"] = {"f1": round(f1, 2), "title": src and src["title"], "url": src and src["url"], "label": "supported" if f1 >= 0.5 else "unverified"} return res def main(argv=None): ap = argparse.ArgumentParser(prog="wiki-model", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("topic", nargs="*", help="topic to state a fact about") ap.add_argument("--q", help="ask a question") ap.add_argument("--fix", help="correct a (possibly false) statement") ap.add_argument("--random", action="store_true", help="a random fact") ap.add_argument("--grounded", action="store_true", help="answer from the sourced Wikipedia index") ap.add_argument("--model", choices=["tiny", "135m"], default=None, help="default: the weights next to this package (a downloaded repo), else tiny") ap.add_argument("--model-path"); ap.add_argument("--index") ap.add_argument("--no-index", action="store_true", help="skip the index (no support check)") ap.add_argument("--n", type=int, default=1, help="number of answers (sampled when > 1)") ap.add_argument("--temperature", type=float, default=None) ap.add_argument("--seed", type=int); ap.add_argument("--device", default="auto") ap.add_argument("--json", action="store_true") a = ap.parse_args(argv) if a.q: task, text = "q", a.q elif a.fix: task, text = "fix", a.fix elif a.random: task, text = "random", None elif a.topic: task, text = "fact", " ".join(a.topic) else: ap.print_help(); return 1 if a.model is None: cfg = ROOT / "config.json" a.model = "135m" if cfg.exists() and "model_type" in json.loads(cfg.read_text()) else "tiny" model = load_model(a.model, a.model_path, a.device) index = None if a.grounded or not a.no_index: try: from .index import FactIndex index = FactIndex(a.index or default_index_path()) except Exception as e: if a.grounded: sys.exit(f"--grounded needs the fact index ({e})") temp = a.temperature if a.temperature is not None else (0.8 if a.n > 1 or task == "random" else 0.0) for i in range(a.n): seed = (a.seed if a.seed is not None else random.randrange(1 << 30)) + i r = run(model, task, text, index, a.grounded, temp, seed) if a.json: print(json.dumps(r, ensure_ascii=False)); continue if task == "random": print(f"[{r['input']}]") print(r["answer"]) for s in r["sources"][:1]: print(f" source: {s['title']} — {s['url']}") if "support" in r: s = r["support"] print(f" {s['label']} (overlap {s['f1']})" + (f" — closest: {s['url']}" if s["url"] else "")) return 0 if __name__ == "__main__": sys.exit(main())