Spaces:
Sleeping
Sleeping
| import { Check, Copy, Layers3 } from "lucide-react"; | |
| import { useMemo, useState, type CSSProperties } from "react"; | |
| import type { TokenizationResult } from "../tokenizer/types"; | |
| const DISPLAY_LIMIT = 2500; | |
| const TOKEN_COLOR_COUNT = 8; | |
| type TokenView = "all" | "special"; | |
| function tokenColor(id: number): string { | |
| return `var(--token-color-${id % TOKEN_COLOR_COUNT})`; | |
| } | |
| export function TokenOutput({ result }: { result: TokenizationResult | null }) { | |
| const [copied, setCopied] = useState(false); | |
| const [view, setView] = useState<TokenView>("all"); | |
| const filteredTokens = useMemo( | |
| () => | |
| result?.tokens.filter((token) => view === "all" || token.special) ?? [], | |
| [result, view], | |
| ); | |
| async function copyIds() { | |
| await navigator.clipboard.writeText( | |
| result?.tokens.map((token) => token.id).join(", ") ?? "", | |
| ); | |
| setCopied(true); | |
| window.setTimeout(() => setCopied(false), 1600); | |
| } | |
| return ( | |
| <section className="surface overflow-hidden" aria-labelledby="output-heading"> | |
| <div className="token-output-header flex flex-wrap items-center justify-between gap-3 px-5 py-3.5"> | |
| <div className="flex items-center gap-2.5"> | |
| <h2 id="output-heading" className="text-sm font-semibold text-[var(--ink)]"> | |
| Tokens | |
| </h2> | |
| <span className="metric-chip"> | |
| <strong>{result?.tokenCount.toLocaleString() ?? 0}</strong> tokens | |
| </span> | |
| </div> | |
| <div className="flex items-center gap-2"> | |
| <label className="flex items-center gap-2 text-[10px] text-[var(--muted)]"> | |
| View | |
| <select | |
| className="select-control" | |
| value={view} | |
| onChange={(event) => setView(event.target.value as TokenView)} | |
| > | |
| <option value="all">全部 Token</option> | |
| <option value="special">Special Tokens</option> | |
| </select> | |
| </label> | |
| <button | |
| className="mini-icon-button" | |
| type="button" | |
| onClick={() => void copyIds()} | |
| disabled={!result} | |
| aria-label="复制所有 Token ID" | |
| title="复制所有 Token ID" | |
| > | |
| {copied ? <Check size={14} /> : <Copy size={14} />} | |
| </button> | |
| </div> | |
| </div> | |
| {!result ? ( | |
| <div className="grid min-h-64 place-items-center px-6 text-center"> | |
| <div> | |
| <span className="mx-auto grid size-11 place-items-center rounded-2xl border border-[var(--line)] bg-[var(--soft)] text-[var(--muted)]"> | |
| <Layers3 size={19} aria-hidden="true" /> | |
| </span> | |
| <p className="mt-3 text-sm font-medium text-[var(--ink)]">等待 Tokenize</p> | |
| <p className="mt-1 text-xs text-[var(--muted)]">Token 文本和 ID 会显示在这里。</p> | |
| </div> | |
| </div> | |
| ) : ( | |
| <div className="p-4 sm:p-5"> | |
| {filteredTokens.length ? ( | |
| <div className="token-stage flex min-h-36 flex-wrap content-start items-start gap-2" aria-label="Token 序列"> | |
| {filteredTokens.slice(0, DISPLAY_LIMIT).map((token) => { | |
| const style = { | |
| "--token-color": tokenColor(token.id), | |
| } as CSSProperties; | |
| return ( | |
| <span | |
| className={`token-chip ${token.special ? "token-chip-special" : ""}`} | |
| key={`${token.index}-${token.id}`} | |
| style={style} | |
| title={`#${token.index} · ID ${token.id}${token.special ? " · special" : ""}`} | |
| > | |
| <span className="token-text">{token.displayText}</span> | |
| <span className="token-id">{token.id}</span> | |
| </span> | |
| ); | |
| })} | |
| </div> | |
| ) : ( | |
| <div className="grid min-h-36 place-items-center text-xs text-[var(--muted)]"> | |
| 当前结果中没有 Special Token。 | |
| </div> | |
| )} | |
| {filteredTokens.length > DISPLAY_LIMIT ? ( | |
| <p className="mt-4 rounded-xl bg-[var(--soft)] px-4 py-3 text-xs text-[var(--muted)]"> | |
| 为保持页面流畅，仅展示前 {DISPLAY_LIMIT.toLocaleString()} 个 Token。 | |
| </p> | |
| ) : null} | |
| <div className="token-map-row"> | |
| <span className="token-map-label">序列概览</span> | |
| <div className="token-map" aria-hidden="true"> | |
| {result.tokens.slice(0, 400).map((token) => ( | |
| <span | |
| className="token-map-segment" | |
| key={`map-${token.index}`} | |
| style={{ | |
| background: tokenColor(token.id), | |
| flexGrow: Math.min(8, Math.max(1, token.text.length)), | |
| }} | |
| /> | |
| ))} | |
| </div> | |
| </div> | |
| </div> | |
| )} | |
| </section> | |
| ); | |
| } | |