Hugging Face's logo

Spaces:
Orion-zhen
/
tokenize-it
Sleeping

App Files Community
tokenize-it
File size: 4,975 Bytes
21ab33c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import { Check, Copy, Layers3 } from "lucide-react";
import { useMemo, useState, type CSSProperties } from "react";

import type { TokenizationResult } from "../tokenizer/types";

const DISPLAY_LIMIT = 2500;
const TOKEN_COLOR_COUNT = 8;
type TokenView = "all" | "special";

function tokenColor(id: number): string {
  return `var(--token-color-${id % TOKEN_COLOR_COUNT})`;
}

export function TokenOutput({ result }: { result: TokenizationResult | null }) {
  const [copied, setCopied] = useState(false);
  const [view, setView] = useState<TokenView>("all");
  const filteredTokens = useMemo(
    () =>
      result?.tokens.filter((token) => view === "all" || token.special) ?? [],
    [result, view],
  );

  async function copyIds() {
    await navigator.clipboard.writeText(
      result?.tokens.map((token) => token.id).join(", ") ?? "",
    );
    setCopied(true);
    window.setTimeout(() => setCopied(false), 1600);
  }

  return (
    <section className="surface overflow-hidden" aria-labelledby="output-heading">
      <div className="token-output-header flex flex-wrap items-center justify-between gap-3 px-5 py-3.5">
        <div className="flex items-center gap-2.5">
          <h2 id="output-heading" className="text-sm font-semibold text-[var(--ink)]">
            Tokens
          </h2>
          <span className="metric-chip">
            <strong>{result?.tokenCount.toLocaleString() ?? 0}</strong> tokens
          </span>
        </div>
        <div className="flex items-center gap-2">
          <label className="flex items-center gap-2 text-[10px] text-[var(--muted)]">
            View
            <select
              className="select-control"
              value={view}
              onChange={(event) => setView(event.target.value as TokenView)}
            >
              <option value="all">全部 Token</option>
              <option value="special">Special Tokens</option>
            </select>
          </label>
          <button
            className="mini-icon-button"
            type="button"
            onClick={() => void copyIds()}
            disabled={!result}
            aria-label="复制所有 Token ID"
            title="复制所有 Token ID"
          >
            {copied ? <Check size={14} /> : <Copy size={14} />}
          </button>
        </div>
      </div>

      {!result ? (
        <div className="grid min-h-64 place-items-center px-6 text-center">
          <div>
            <span className="mx-auto grid size-11 place-items-center rounded-2xl border border-[var(--line)] bg-[var(--soft)] text-[var(--muted)]">
              <Layers3 size={19} aria-hidden="true" />
            </span>
            <p className="mt-3 text-sm font-medium text-[var(--ink)]">等待 Tokenize</p>
            <p className="mt-1 text-xs text-[var(--muted)]">Token 文本和 ID 会显示在这里。</p>
          </div>
        </div>
      ) : (
        <div className="p-4 sm:p-5">
          {filteredTokens.length ? (
            <div className="token-stage flex min-h-36 flex-wrap content-start items-start gap-2" aria-label="Token 序列">
              {filteredTokens.slice(0, DISPLAY_LIMIT).map((token) => {
                const style = {
                  "--token-color": tokenColor(token.id),
                } as CSSProperties;
                return (
                  <span
                    className={`token-chip ${token.special ? "token-chip-special" : ""}`}
                    key={`${token.index}-${token.id}`}
                    style={style}
                    title={`#${token.index} · ID ${token.id}${token.special ? " · special" : ""}`}
                  >
                    <span className="token-text">{token.displayText}</span>
                    <span className="token-id">{token.id}</span>
                  </span>
                );
              })}
            </div>
          ) : (
            <div className="grid min-h-36 place-items-center text-xs text-[var(--muted)]">
              当前结果中没有 Special Token。
            </div>
          )}

          {filteredTokens.length > DISPLAY_LIMIT ? (
            <p className="mt-4 rounded-xl bg-[var(--soft)] px-4 py-3 text-xs text-[var(--muted)]">
              为保持页面流畅，仅展示前 {DISPLAY_LIMIT.toLocaleString()} 个 Token。
            </p>
          ) : null}

          <div className="token-map-row">
            <span className="token-map-label">序列概览</span>
            <div className="token-map" aria-hidden="true">
              {result.tokens.slice(0, 400).map((token) => (
                <span
                  className="token-map-segment"
                  key={`map-${token.index}`}
                  style={{
                    background: tokenColor(token.id),
                    flexGrow: Math.min(8, Math.max(1, token.text.length)),
                  }}
                />
              ))}
            </div>
          </div>
        </div>
      )}
    </section>
  );
}