Hugging Face's logo

Spaces:
Orion-zhen
/
tokenize-it
Sleeping

App Files Community
tokenize-it
File size: 3,625 Bytes
21ab33c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
import { Eraser, LoaderCircle, Sparkles, WandSparkles } from "lucide-react";

const EXAMPLE = "Tokenization turns language into structure. 理解分词，就是理解模型如何阅读世界。";

interface InputPanelProps {
  text: string;
  ready: boolean;
  loading: boolean;
  tokenizing: boolean;
  dirtyModel: boolean;
  onTextChange: (value: string) => void;
  onTokenize: () => void;
}

export function InputPanel({
  text,
  ready,
  loading,
  tokenizing,
  dirtyModel,
  onTextChange,
  onTokenize,
}: InputPanelProps) {
  const words = text.trim() ? text.trim().split(/\s+/u).length : 0;
  const lines = text ? text.split(/\r?\n/u).length : 0;
  const disabled = !ready || loading || tokenizing || !text || dirtyModel;

  return (
    <section className="surface p-3" aria-labelledby="input-heading">
      <div className="flex items-center justify-between px-2 pb-2">
        <h1 id="input-heading" className="text-sm font-semibold text-[var(--ink)]">
          输入
        </h1>
        <span className="eyebrow">Local only</span>
      </div>

      <div className="editor-shell">
        <textarea
          className="min-h-48 w-full resize-y bg-transparent px-4 pb-3 pt-4 text-[16px] leading-7 text-[var(--ink)] outline-none placeholder:text-[var(--muted)] sm:min-h-52"
          value={text}
          maxLength={200000}
          onChange={(event) => onTextChange(event.target.value)}
          onKeyDown={(event) => {
            if ((event.metaKey || event.ctrlKey) && event.key === "Enter") {
              event.preventDefault();
              onTokenize();
            }
          }}
          placeholder="写下或粘贴一段文本……"
          aria-label="需要 Tokenize 的文本"
        />

        <div className="flex flex-col gap-2 border-t border-[var(--line)] px-3 py-2.5 sm:flex-row sm:items-center sm:justify-between">
          <span className="font-mono text-[10px] text-[var(--muted)]">
            {text.length.toLocaleString()} characters&nbsp; · &nbsp;{words.toLocaleString()} words&nbsp; · &nbsp;{lines} lines
          </span>
          <div className="flex items-center justify-end gap-1.5">
            <span className="mr-1 hidden text-[9px] text-[var(--muted)] md:inline">
              ⌘ / Ctrl + Enter
            </span>
            <button
              className="mini-icon-button"
              type="button"
              onClick={() => onTextChange(EXAMPLE)}
              aria-label="填入示例文本"
              title="示例文本"
            >
              <WandSparkles size={14} />
            </button>
            <button
              className="mini-icon-button"
              type="button"
              onClick={() => onTextChange("")}
              aria-label="清空文本"
              title="清空"
            >
              <Eraser size={14} />
            </button>
            <button
              className="tokenize-button ml-1 min-w-32 justify-center"
              type="button"
              onClick={onTokenize}
              disabled={disabled}
            >
              {tokenizing ? (
                <LoaderCircle className="animate-spin" size={16} />
              ) : (
                <Sparkles size={16} />
              )}
              {dirtyModel
                ? "重新加载模型"
                : loading
                  ? "加载中"
                  : tokenizing
                    ? "解析中"
                    : ready
                      ? "Tokenize"
                      : "先加载模型"}
            </button>
          </div>
        </div>
      </div>
    </section>
  );
}