import { Check, Copy, Layers3 } from "lucide-react"; import { useMemo, useState, type CSSProperties } from "react"; import type { TokenizationResult } from "../tokenizer/types"; const DISPLAY_LIMIT = 2500; const TOKEN_COLOR_COUNT = 8; type TokenView = "all" | "special"; function tokenColor(id: number): string { return `var(--token-color-${id % TOKEN_COLOR_COUNT})`; } export function TokenOutput({ result }: { result: TokenizationResult | null }) { const [copied, setCopied] = useState(false); const [view, setView] = useState("all"); const filteredTokens = useMemo( () => result?.tokens.filter((token) => view === "all" || token.special) ?? [], [result, view], ); async function copyIds() { await navigator.clipboard.writeText( result?.tokens.map((token) => token.id).join(", ") ?? "", ); setCopied(true); window.setTimeout(() => setCopied(false), 1600); } return (

Tokens

{result?.tokenCount.toLocaleString() ?? 0} tokens
{!result ? (

等待 Tokenize

Token 文本和 ID 会显示在这里。

) : (
{filteredTokens.length ? (
{filteredTokens.slice(0, DISPLAY_LIMIT).map((token) => { const style = { "--token-color": tokenColor(token.id), } as CSSProperties; return ( {token.displayText} {token.id} ); })}
) : (
当前结果中没有 Special Token。
)} {filteredTokens.length > DISPLAY_LIMIT ? (

为保持页面流畅,仅展示前 {DISPLAY_LIMIT.toLocaleString()} 个 Token。

) : null}
序列概览
)}
); }