Text Generation
Transformers
Safetensors
English
llama
wikipedia
facts
retrieval
question-answering
smollm2
text-generation-inference
Lucasgitguy-co's picture
wiki-model 0.1.0
4f2ae40 verified
Raw History Blame Contribute Delete
9.64 kB
"""wiki-model command line.
wiki-model "Eiffel Tower" a fact about a topic
wiki-model --random a random fact
wiki-model --q "Who wrote Hamlet?" question answering
wiki-model --fix "The Moon is made of cheese." correct a false statement
add --grounded to answer from the sourced Wikipedia index (shows the source URL)
--model tiny|135m (default: tiny)
"""
import argparse, json, os, random, re, sys
from pathlib import Path
from .prompts import fact, question, fix, CANT
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
def _device(want):
import torch
if want != "auto": return want
return "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu"
class Tiny:
name = "wiki-model-tiny"
def __init__(self, path, device="cpu"):
from tokenizers import Tokenizer
from .model import TinyGPT
self.m = TinyGPT.load(path, device)
tj = Path(path) / "tokenizer.json"
self.tok = Tokenizer.from_file(str(tj if tj.exists() else HERE / "tok" / "tokenizer.json"))
self.eos = self.tok.token_to_id("<eos>")
def __call__(self, prompt, max_new=80, temperature=0.0, seed=None, stop=None):
import torch
g = torch.Generator().manual_seed(seed) if seed is not None else None
ids = self.tok.encode(prompt).ids
out = self.tok.decode(self.m.generate(ids, max_new, self.eos, temperature, gen=g), skip_special_tokens=False)
if stop and stop in out: out = out.split(stop)[0]
return out.replace("<eos>", "").strip()
class _AnswerNoRepeat:
"""Block 4-gram repeats inside the generated answer only. (transformers' no_repeat_ngram_size also
counts the prompt, which forbids copying a claim or a value from <ctx>.)"""
def __init__(self, prompt_len, n=4):
self.p, self.n = prompt_len, n
def __call__(self, input_ids, scores):
for b, seq in enumerate(input_ids[:, self.p:].tolist()):
if len(seq) < self.n - 1: continue
tail = tuple(seq[-(self.n - 1):])
banned = [seq[i + self.n - 1] for i in range(len(seq) - self.n + 1) if tuple(seq[i:i + self.n - 1]) == tail]
if banned: scores[b, banned] = float("-inf")
return scores
class Smol:
name = "wiki-model"
def __init__(self, path, device="cpu"):
import logging
logging.getLogger("torch.distributed.elastic.multiprocessing.redirects").setLevel(logging.ERROR)
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.utils import logging as tlog
tlog.set_verbosity_error(); tlog.disable_progress_bar()
self.tok = AutoTokenizer.from_pretrained(path)
try:
m = AutoModelForCausalLM.from_pretrained(path, dtype=torch.float32)
except TypeError: # transformers < 4.56 only knows torch_dtype
m = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.float32)
self.m = m.to(device).eval()
self.dev = device
def __call__(self, prompt, max_new=80, temperature=0.0, seed=None, stop=None):
import torch
if seed is not None: torch.manual_seed(seed)
x = self.tok(prompt, return_tensors="pt", add_special_tokens=False).to(self.dev)
kw = dict(do_sample=True, temperature=temperature, top_p=0.9) if temperature > 0 else dict(do_sample=False)
with torch.no_grad():
y = self.m.generate(**x, max_new_tokens=max_new, eos_token_id=self.tok.eos_token_id,
logits_processor=[_AnswerNoRepeat(x["input_ids"].shape[1])],
pad_token_id=self.tok.eos_token_id, **kw)
out = self.tok.decode(y[0, x["input_ids"].shape[1]:], skip_special_tokens=True)
if stop and stop in out: out = out.split(stop)[0]
return out.strip()
def default_model_path(kind):
env = os.environ.get("WIKI_MODEL_TINY" if kind == "tiny" else "WIKI_MODEL_135M")
if env: return env
cfg = ROOT / "config.json" # running from inside a downloaded HF repo (model files next to wiki_model/)
if cfg.exists() and (ROOT / "model.safetensors").exists():
import json
if ("model_type" in json.load(open(cfg))) == (kind == "135m"): return str(ROOT)
local = ROOT / "runs" / ("tiny-sft" if kind == "tiny" else "smol-sft") / "final"
if local.exists(): return str(local)
from huggingface_hub import snapshot_download
return snapshot_download("Lucasgitguy-co/" + ("wiki-model-tiny" if kind == "tiny" else "wiki-model"))
def default_index_path():
env = os.environ.get("WIKI_MODEL_INDEX")
if env: return env
local = ROOT / "data" / "index"
if (local / "docs.sqlite").exists(): return str(local)
from huggingface_hub import snapshot_download
return str(Path(snapshot_download("Lucasgitguy-co/wiki-model-facts", repo_type="dataset",
allow_patterns=["index/*", "index/bm25/*"])) / "index")
def load_model(kind, path=None, device="auto"):
dev = _device(device)
return (Tiny if kind == "tiny" else Smol)(path or default_model_path(kind), dev)
def context_for(index, query, topic=None):
"""Same shape as training <ctx>: best article text, optionally ' | ' + a shorter second hit."""
hits = []
if topic and (d := index.by_title(topic)): hits.append(d)
for h in index.search(query, 3):
if all(h["title"] != x["title"] for x in hits): hits.append(h)
if not hits: return None, []
ctx = hits[0]["text"]
if len(hits) > 1: ctx += " | " + hits[1]["text"][:250]
return ctx, hits[:2]
def run(model, task, text, index=None, grounded=False, temperature=0.0, seed=None):
"""Returns dict(answer, sources, support). task in fact|q|fix|random."""
rng = random.Random(seed)
if task == "random": # a real topic (index, or the bundled list of training topics); the model states the fact
if index is not None:
text = index.random(rng)["title"]
else:
text = rng.choice((HERE / "topics.txt").read_text().split("\n")[:-1])
task = "fact"
ctx, hits = (context_for(index, text, topic=text if task == "fact" else None) if grounded and index else (None, []))
prompt = {"fact": fact, "q": question, "fix": fix}[task](text, ctx)
ans = model(prompt, max_new=100, temperature=temperature, seed=seed)
ans = re.sub(r"\s+", " ", ans).strip()
res = {"task": task, "input": text, "answer": ans, "sources": [{"title": h["title"], "url": h["url"]} for h in hits]}
if index is not None and not grounded and task in ("fact", "fix") and ans and ans != CANT:
f1, src = index.support(ans, topic=text if task == "fact" else None)
res["support"] = {"f1": round(f1, 2), "title": src and src["title"], "url": src and src["url"],
"label": "supported" if f1 >= 0.5 else "unverified"}
return res
def main(argv=None):
ap = argparse.ArgumentParser(prog="wiki-model", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("topic", nargs="*", help="topic to state a fact about")
ap.add_argument("--q", help="ask a question")
ap.add_argument("--fix", help="correct a (possibly false) statement")
ap.add_argument("--random", action="store_true", help="a random fact")
ap.add_argument("--grounded", action="store_true", help="answer from the sourced Wikipedia index")
ap.add_argument("--model", choices=["tiny", "135m"], default=None,
help="default: the weights next to this package (a downloaded repo), else tiny")
ap.add_argument("--model-path"); ap.add_argument("--index")
ap.add_argument("--no-index", action="store_true", help="skip the index (no support check)")
ap.add_argument("--n", type=int, default=1, help="number of answers (sampled when > 1)")
ap.add_argument("--temperature", type=float, default=None)
ap.add_argument("--seed", type=int); ap.add_argument("--device", default="auto")
ap.add_argument("--json", action="store_true")
a = ap.parse_args(argv)
if a.q: task, text = "q", a.q
elif a.fix: task, text = "fix", a.fix
elif a.random: task, text = "random", None
elif a.topic: task, text = "fact", " ".join(a.topic)
else: ap.print_help(); return 1
if a.model is None:
cfg = ROOT / "config.json"
a.model = "135m" if cfg.exists() and "model_type" in json.loads(cfg.read_text()) else "tiny"
model = load_model(a.model, a.model_path, a.device)
index = None
if a.grounded or not a.no_index:
try:
from .index import FactIndex
index = FactIndex(a.index or default_index_path())
except Exception as e:
if a.grounded: sys.exit(f"--grounded needs the fact index ({e})")
temp = a.temperature if a.temperature is not None else (0.8 if a.n > 1 or task == "random" else 0.0)
for i in range(a.n):
seed = (a.seed if a.seed is not None else random.randrange(1 << 30)) + i
r = run(model, task, text, index, a.grounded, temp, seed)
if a.json: print(json.dumps(r, ensure_ascii=False)); continue
if task == "random": print(f"[{r['input']}]")
print(r["answer"])
for s in r["sources"][:1]: print(f" source: {s['title']} — {s['url']}")
if "support" in r:
s = r["support"]
print(f" {s['label']} (overlap {s['f1']})" + (f" — closest: {s['url']}" if s["url"] else ""))
return 0
if __name__ == "__main__":
sys.exit(main())