Tetu-2 / chat.py
tetu-AI's picture
Initial release
c9f4679 verified
Raw History Blame Contribute Delete
3.14 kB
"""Chat with T锚tu-2 in the terminal (it speaks French). Each word is colored by its doubt (green: sure,
yellow: hesitates, red: in doubt). Commands: /reset, /temp 0.5, /color, /quit.
python chat.py (GPU)
python chat.py --cpu (no GPU, slower)
python chat.py --no-calc (turn the built-in calculator off)
"""
import argparse
import os
from pathlib import Path
import sys
import time
ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT))
import torch
from tetu.chat import ChatEncoder
from tetu2.generate import Generator, load
GREEN, YELLOW, RED, BOLD, DIM, RESET = "\033[32m", "\033[33m", "\033[31m", "\033[1m", "\033[2m", "\033[0m"
def color(d):
return GREEN if d < 0.2 else YELLOW if d < 0.45 else RED
def main():
p = argparse.ArgumentParser()
p.add_argument("--ckpt", default=str(ROOT))
p.add_argument("--cpu", action="store_true")
p.add_argument("--no-calc", dest="calc", action="store_false")
p.add_argument("--temp", type=float, default=0.3)
p.add_argument("--max_new", type=int, default=600)
p.add_argument("--max_len", type=int, default=8192)
a = p.parse_args()
if sys.platform == "win32":
os.system("")
dev = "cpu" if a.cpu or not torch.cuda.is_available() else "cuda"
model, tok = load(a.ckpt, device=dev, dtype=torch.float32 if dev == "cpu" else torch.bfloat16)
enc = ChatEncoder(tok)
gen = Generator(model, tok, max_len=a.max_len)
print(f"{BOLD}T锚tu-2{RESET} {DIM}路 /reset 路 /temp x 路 /color 路 /quit{RESET}\n")
turns, temp, use_color = [], a.temp, True
while True:
try:
q = input(f"{BOLD}you>{RESET} ").strip()
except (EOFError, KeyboardInterrupt):
break
if not q:
continue
if q in ("/quit", "/q", "/exit"):
break
if q == "/reset":
turns = []
print(f"{DIM}new conversation{RESET}\n")
continue
if q.startswith("/temp"):
temp = float(q.split()[1])
continue
if q == "/color":
use_color = not use_color
continue
turns.append(("user", q))
ids = enc.encode(turns, generation_prompt=True)[0]
while len(ids) > a.max_len - a.max_new and len(turns) > 1:
turns = turns[2:]
ids = enc.encode(turns, generation_prompt=True)[0]
print(f"{BOLD}t锚tu>{RESET} ", end="", flush=True)
t0, pieces, doubts = time.time(), [], []
kw = dict(rep=1.0, calc=True, presence=0.5, frequency=0.3) if a.calc else dict(rep=1.05)
for piece, d in gen.stream(ids=ids, max_new=a.max_new, temp=temp, top_k=40, **kw):
pieces.append(piece)
doubts.append(d)
print((color(d) if use_color else "") + piece + (RESET if use_color else ""), end="", flush=True)
dt = time.time() - t0
turns.append(("assistant", "".join(pieces).strip()))
print(f"\n{DIM}[{len(doubts)} tokens 路 {len(doubts) / max(dt, 1e-6):.0f} tok/s]{RESET}\n")
if __name__ == "__main__":
with torch.inference_mode():
main()