Download chat.py from tetu-AI/Tetu-2: direct link, hf CLI and curl.
- Browser
- Download file 3.14 kB
-
https://huggingface.co/tetu-AI/Tetu-2/resolve/main/chat.py
- Command line
-
hf download hf://tetu-AI/Tetu-2/chat.py
-
curl -L -o chat.py https://huggingface.co/tetu-AI/Tetu-2/resolve/main/chat.py
3.14 kB
| """Chat with T锚tu-2 in the terminal (it speaks French). Each word is colored by its doubt (green: sure, | |
| yellow: hesitates, red: in doubt). Commands: /reset, /temp 0.5, /color, /quit. | |
| python chat.py (GPU) | |
| python chat.py --cpu (no GPU, slower) | |
| python chat.py --no-calc (turn the built-in calculator off) | |
| """ | |
| import argparse | |
| import os | |
| from pathlib import Path | |
| import sys | |
| import time | |
| ROOT = Path(__file__).resolve().parent | |
| sys.path.insert(0, str(ROOT)) | |
| import torch | |
| from tetu.chat import ChatEncoder | |
| from tetu2.generate import Generator, load | |
| GREEN, YELLOW, RED, BOLD, DIM, RESET = "\033[32m", "\033[33m", "\033[31m", "\033[1m", "\033[2m", "\033[0m" | |
| def color(d): | |
| return GREEN if d < 0.2 else YELLOW if d < 0.45 else RED | |
| def main(): | |
| p = argparse.ArgumentParser() | |
| p.add_argument("--ckpt", default=str(ROOT)) | |
| p.add_argument("--cpu", action="store_true") | |
| p.add_argument("--no-calc", dest="calc", action="store_false") | |
| p.add_argument("--temp", type=float, default=0.3) | |
| p.add_argument("--max_new", type=int, default=600) | |
| p.add_argument("--max_len", type=int, default=8192) | |
| a = p.parse_args() | |
| if sys.platform == "win32": | |
| os.system("") | |
| dev = "cpu" if a.cpu or not torch.cuda.is_available() else "cuda" | |
| model, tok = load(a.ckpt, device=dev, dtype=torch.float32 if dev == "cpu" else torch.bfloat16) | |
| enc = ChatEncoder(tok) | |
| gen = Generator(model, tok, max_len=a.max_len) | |
| print(f"{BOLD}T锚tu-2{RESET} {DIM}路 /reset 路 /temp x 路 /color 路 /quit{RESET}\n") | |
| turns, temp, use_color = [], a.temp, True | |
| while True: | |
| try: | |
| q = input(f"{BOLD}you>{RESET} ").strip() | |
| except (EOFError, KeyboardInterrupt): | |
| break | |
| if not q: | |
| continue | |
| if q in ("/quit", "/q", "/exit"): | |
| break | |
| if q == "/reset": | |
| turns = [] | |
| print(f"{DIM}new conversation{RESET}\n") | |
| continue | |
| if q.startswith("/temp"): | |
| temp = float(q.split()[1]) | |
| continue | |
| if q == "/color": | |
| use_color = not use_color | |
| continue | |
| turns.append(("user", q)) | |
| ids = enc.encode(turns, generation_prompt=True)[0] | |
| while len(ids) > a.max_len - a.max_new and len(turns) > 1: | |
| turns = turns[2:] | |
| ids = enc.encode(turns, generation_prompt=True)[0] | |
| print(f"{BOLD}t锚tu>{RESET} ", end="", flush=True) | |
| t0, pieces, doubts = time.time(), [], [] | |
| kw = dict(rep=1.0, calc=True, presence=0.5, frequency=0.3) if a.calc else dict(rep=1.05) | |
| for piece, d in gen.stream(ids=ids, max_new=a.max_new, temp=temp, top_k=40, **kw): | |
| pieces.append(piece) | |
| doubts.append(d) | |
| print((color(d) if use_color else "") + piece + (RESET if use_color else ""), end="", flush=True) | |
| dt = time.time() - t0 | |
| turns.append(("assistant", "".join(pieces).strip())) | |
| print(f"\n{DIM}[{len(doubts)} tokens 路 {len(doubts) / max(dt, 1e-6):.0f} tok/s]{RESET}\n") | |
| if __name__ == "__main__": | |
| with torch.inference_mode(): | |
| main() | |