"""Chat with Têtu-2 in the terminal (it speaks French). Each word is colored by its doubt (green: sure, yellow: hesitates, red: in doubt). Commands: /reset, /temp 0.5, /color, /quit. python chat.py (GPU) python chat.py --cpu (no GPU, slower) python chat.py --no-calc (turn the built-in calculator off) """ import argparse import os from pathlib import Path import sys import time ROOT = Path(__file__).resolve().parent sys.path.insert(0, str(ROOT)) import torch from tetu.chat import ChatEncoder from tetu2.generate import Generator, load GREEN, YELLOW, RED, BOLD, DIM, RESET = "\033[32m", "\033[33m", "\033[31m", "\033[1m", "\033[2m", "\033[0m" def color(d): return GREEN if d < 0.2 else YELLOW if d < 0.45 else RED def main(): p = argparse.ArgumentParser() p.add_argument("--ckpt", default=str(ROOT)) p.add_argument("--cpu", action="store_true") p.add_argument("--no-calc", dest="calc", action="store_false") p.add_argument("--temp", type=float, default=0.3) p.add_argument("--max_new", type=int, default=600) p.add_argument("--max_len", type=int, default=8192) a = p.parse_args() if sys.platform == "win32": os.system("") dev = "cpu" if a.cpu or not torch.cuda.is_available() else "cuda" model, tok = load(a.ckpt, device=dev, dtype=torch.float32 if dev == "cpu" else torch.bfloat16) enc = ChatEncoder(tok) gen = Generator(model, tok, max_len=a.max_len) print(f"{BOLD}Têtu-2{RESET} {DIM}· /reset · /temp x · /color · /quit{RESET}\n") turns, temp, use_color = [], a.temp, True while True: try: q = input(f"{BOLD}you>{RESET} ").strip() except (EOFError, KeyboardInterrupt): break if not q: continue if q in ("/quit", "/q", "/exit"): break if q == "/reset": turns = [] print(f"{DIM}new conversation{RESET}\n") continue if q.startswith("/temp"): temp = float(q.split()[1]) continue if q == "/color": use_color = not use_color continue turns.append(("user", q)) ids = enc.encode(turns, generation_prompt=True)[0] while len(ids) > a.max_len - a.max_new and len(turns) > 1: turns = turns[2:] ids = enc.encode(turns, generation_prompt=True)[0] print(f"{BOLD}têtu>{RESET} ", end="", flush=True) t0, pieces, doubts = time.time(), [], [] kw = dict(rep=1.0, calc=True, presence=0.5, frequency=0.3) if a.calc else dict(rep=1.05) for piece, d in gen.stream(ids=ids, max_new=a.max_new, temp=temp, top_k=40, **kw): pieces.append(piece) doubts.append(d) print((color(d) if use_color else "") + piece + (RESET if use_color else ""), end="", flush=True) dt = time.time() - t0 turns.append(("assistant", "".join(pieces).strip())) print(f"\n{DIM}[{len(doubts)} tokens · {len(doubts) / max(dt, 1e-6):.0f} tok/s]{RESET}\n") if __name__ == "__main__": with torch.inference_mode(): main()