Buckets:

hf-doc-build
/

doc

Files
xet
hf-doc-build/doc / transformers /main /ko /tokenizer_summary.html
mishig
about 12 hours ago
download
raw
43.7 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;토큰화 알고리즘&quot;,&quot;local&quot;:&quot;tokenization-algorithms&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;바이트 쌍 인코딩(BPE)&quot;,&quot;local&quot;:&quot;byte-pair-encoding-bpe&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;바이트 수준 BPE&quot;,&quot;local&quot;:&quot;byte-level-bpe&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Unigram&quot;,&quot;local&quot;:&quot;unigram&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;SentencePiece&quot;,&quot;local&quot;:&quot;sentencepiece&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;WordPiece&quot;,&quot;local&quot;:&quot;wordpiece&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;단어 수준 토큰화&quot;,&quot;local&quot;:&quot;word-level-tokenization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;문자 단위 토큰화&quot;,&quot;local&quot;:&quot;character-level-tokenization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;리소스&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/transformers/main/en/_app/immutable/entry/start.ByZEd4qm.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DHdg-_0Z.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/CBySwR1V.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/entry/app.CQqZS61p.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/Cdb1BouH.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/dUGfULTE.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DOLBYqcr.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/nodes/0.CEy2b_T8.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/Conj1S5Z.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/nodes/766.COOYlBsX.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DoQ7MZQh.js" rel="modulepreload">
		<link href="/docs/transformers/main/en/_app/immutable/chunks/DnCf4Kx5.js" rel="modulepreload">
		<!--klul1d--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Tokenization algorithms&quot;,&quot;local&quot;:&quot;tokenization-algorithms&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Byte pair encoding (BPE)&quot;,&quot;local&quot;:&quot;byte-pair-encoding-bpe&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Byte-level BPE&quot;,&quot;local&quot;:&quot;byte-level-bpe&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Unigram&quot;,&quot;local&quot;:&quot;unigram&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;SentencePiece&quot;,&quot;local&quot;:&quot;sentencepiece&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;WordPiece&quot;,&quot;local&quot;:&quot;wordpiece&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Word-level tokenization&quot;,&quot;local&quot;:&quot;word-level-tokenization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Character-level tokenization&quot;,&quot;local&quot;:&quot;character-level-tokenization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Resources&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/transformers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="tokenization-algorithms" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenization-algorithms"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->토큰화 알고리즘<!----></span></h1><!--]--><!----> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/zHvTiHr506c" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p>Transformers는 세 가지 서브워드 토큰화 알고리즘을 지원합니다: Byte pair encoding (BPE), Unigram, WordPiece. 이들은 텍스트를 단어와 문자 사이의 단위로 분할하여, 의미 있는 조각을 포착하면서도 어휘를 간결하게 유지합니다. 일반적인 단어는 단일 토큰으로 온전히 유지되고, 드물거나 알려지지 않은 단어는 서브워드로 분해됩니다.</p> <p>예를 들어, <code>annoyingly</code>는 어휘에 따라 <code>["annoying", "ly"]</code> 또는 <code>["annoy", "ing", "ly"]</code>로 분할될 수 있습니다. 서브워드 분할을 통해 모델은 알려진 서브워드를 조합하여 이전에 보지 못한 단어를 표현할 수 있습니다.</p> <blockquote class="tip"><p>Subword tokenization은 터키어처럼 서브워드를 연결하여 길고 복잡한 단어를 형성할 수 있는 언어에서 특히 유용합니다.</p></blockquote> <!--[1--><h2 class="relative group"><a id="byte-pair-encoding-bpe" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#byte-pair-encoding-bpe"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->바이트 쌍 인코딩(BPE)<!----></span></h2><!--]--><!----> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/HEikzVL-lZU" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p><a href="https://huggingface.co/papers/1508.07909" rel="nofollow">Byte pair encoding</a>(BPE)은 Transformers에서 가장 인기 있는 토큰화 알고리즘으로, <a href="./model_doc/llama">Llama</a>, <a href="./model_doc/gemma">Gemma</a>, <a href="./model_doc/qwen2">Qwen2</a> 등 다양한 모델에서 사용됩니다.</p> <ol><li>Pre-tokenizer는 공백이나 기타 규칙에 따라 텍스트를 분할하여 고유 단어 집합과 그 빈도를 생성합니다.</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->(&quot;hug&quot;, 10), (&quot;pug&quot;, 5), (&quot;pun&quot;, 12), (&quot;bun&quot;, 4), (&quot;hugs&quot;, 5)<!----></pre></div><!----> <ol start="2"><li>BPE 알고리즘은 모든 문자에서 기본 어휘인 <code>["b", "g", "h", "n", "p", "s", "u"]</code>를 생성합니다.</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->(&quot;h&quot; &quot;u&quot; &quot;g&quot;, 10), (&quot;p&quot; &quot;u&quot; &quot;g&quot;, 5), (&quot;p&quot; &quot;u&quot; &quot;n&quot;, 12), (&quot;b&quot; &quot;u&quot; &quot;n&quot;, 4), (&quot;h&quot; &quot;u&quot; &quot;g&quot; &quot;s&quot;, 5)<!----></pre></div><!----> <ol start="3"><li>BPE는 개별 문자에서 시작하여 가장 빈번하게 나타나는 인접한 쌍을 반복적으로 병합합니다. <code>"u"</code>와 <code>"g"</code>는 <code>"hug"</code>, <code>"pug"</code>, <code>"hugs"</code>에서 가장 자주 함께 나타나므로, BPE는 이들을 <code>"ug"</code>로 병합하고 이를 어휘에 추가합니다.</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->(&quot;h&quot; &quot;ug&quot;, 10), (&quot;p&quot; &quot;ug&quot;, 5), (&quot;p&quot; &quot;u&quot; &quot;n&quot;, 12), (&quot;b&quot; &quot;u&quot; &quot;n&quot;, 4), (&quot;h&quot; &quot;ug&quot; &quot;s&quot;, 5)<!----></pre></div><!----> <ol start="4"><li>다음으로 흔한 쌍은 <code>"u"</code>와 <code>"n"</code>이며, <code>"pun"</code>과 <code>"bun"</code>에 나타나므로 <code>"un"</code>으로 병합됩니다.</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->(&quot;h&quot; &quot;ug&quot;, 10), (&quot;p&quot; &quot;ug&quot;, 5), (&quot;p&quot; &quot;un&quot;, 12), (&quot;b&quot; &quot;un&quot;, 4), (&quot;h&quot; &quot;ug&quot; &quot;s&quot;, 5)<!----></pre></div><!----> <ol start="5"><li>어휘는 이제 <code>["b", "g", "h", "n", "p", "s", "u", "ug", "un"]</code>입니다. BPE는 기본 어휘 크기加上 병합 횟수에 해당하는 목표 어휘 크기에 도달할 때까지 병합 규칙을 계속 학습합니다. <a href="model_doc/openai-gpt">GPT</a>는 40,478개의 어휘 크기(기본 토큰 478개 + 병합 40,000개)를 가진 BPE를 사용합니다.</li></ol> <p>기본 어휘에 없는 모든 문자는 <code>"&lt;unk>"</code>와 같은 unknown token으로 매핑됩니다. 실제로 기본 어휘는 학습 중에 확인된 모든 문자를 포함하므로, unknown token은 드물게 나타납니다.</p> <!--[2--><h3 class="relative group"><a id="byte-level-bpe" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#byte-level-bpe"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->바이트 수준 BPE<!----></span></h3><!--]--><!----> <p>모든 Unicode 문자를 포함하면 기본 어휘가 매우 커집니다. Byte-level BPE는 대신 256개의 byte 값을 기본 어휘로 사용하여 <code>"&lt;unk>"</code> 토큰 없이 모든 단어를 토큰화할 수 있도록 보장합니다. <a href="./model_doc/gpt2">GPT-2</a>는 50,257개의 어휘 크기(256개 byte 토큰 + 50,000개 merges + 특수 end-of-text 토큰)를 가진 byte-level BPE를 사용합니다.</p> <!--[1--><h2 class="relative group"><a id="unigram" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#unigram"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Unigram<!----></span></h2><!--]--><!----> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/TGZfZVuF9Yc" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p><a href="https://huggingface.co/papers/1804.10959" rel="nofollow">Unigram</a>은 Transformers에서 두 번째로 인기 있는 토큰화 알고리즘으로, <a href="./model_doc/t5">T5</a>, <a href="./model_doc/big_bird">BigBird</a>, <a href="./model_doc/pegasus">Pegasus</a> 등 다양한 모델에서 사용됩니다.</p> <ol><li>Unigram은 대규모 후보 부분어휘 집합으로 시작하며, 각 후보는 해당 부분어휘가 나타나는 빈도에 기반한 확률 점수를 부여받습니다.</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->(&quot;hug&quot;, 10), (&quot;pug&quot;, 5), (&quot;pun&quot;, 12), (&quot;bun&quot;, 4), (&quot;hugs&quot;, 5)
[&quot;b&quot;, &quot;g&quot;, &quot;h&quot;, &quot;n&quot;, &quot;p&quot;, &quot;s&quot;, &quot;u&quot;, &quot;hu&quot;, &quot;ug&quot;, &quot;un&quot;, &quot;pu&quot;, &quot;bu&quot;, &quot;gs&quot;, &quot;hug&quot;, &quot;pug&quot;, &quot;pun&quot;, &quot;bun&quot;, &quot;ugs&quot;, &quot;hugs&quot;]<!----></pre></div><!----> <ol start="2"><li><p>Unigram은 각 단계에서 현재 어휘가 학습 데이터를 얼마나 잘 토큰화하는지 점수를 매깁니다.</p></li> <li><p>각 토큰에 대해 Unigram은 해당 토큰을 제거했을 때 전체 손실이 얼마나 증가하는지 측정합니다. 예를 들어, <code>"pu"</code>를 제거해도 손실에 거의 영향을 미치지 않습니다. <code>"pug"</code>과 <code>"pun"</code>은 여전히 <code>["p", "ug"]</code>와 <code>["p", "un"]</code>로 토큰화될 수 있기 때문입니다.</p> <p>하지만 <code>"ug"</code>를 제거하면 <code>"hug"</code>, <code>"pug"</code>, <code>"hugs"</code>가 모두 이 토큰에 의존하고 있기 때문에 손실이 크게 증가합니다.</p></li> <li><p>Unigram은 손실 증가량이 가장 낮은 토큰, 일반적으로 하위 10~20%를 제거합니다. 기본 문자는 항상 유지되므로 모든 단어를 토큰화할 수 있습니다. <code>"bu"</code>, <code>"pu"</code>, <code>"gs"</code>, <code>"pug"</code>, <code>"bun"</code>와 같은 토큰은 전체 확률에 기여도가 가장 낮아 제거됩니다.</p></li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->[&quot;b&quot;, &quot;g&quot;, &quot;h&quot;, &quot;n&quot;, &quot;p&quot;, &quot;s&quot;, &quot;u&quot;, &quot;hu&quot;, &quot;ug&quot;, &quot;un&quot;, &quot;hug&quot;, &quot;pun&quot;, &quot;ugs&quot;, &quot;hugs&quot;]<!----></pre></div><!----> <ol start="5"><li>단계 2~4는 어휘가 목표 크기에 도달할 때까지 반복됩니다.</li></ol> <p>추론 단계에서 Unigram은 한 단어를 여러 방식으로 토큰화할 수 있습니다. <code>"hugs"</code>는 <code>["hug", "s"]</code>, <code>["h", "ug", "s"]</code>, 또는 <code>["h", "u", "g", "s"]</code>로 토큰화될 수 있습니다. Unigram은 가장 높은 확률을 가진 토큰화를 선택합니다. 병합 규칙에 기반한 결정론적인 BPE와 달리, Unigram은 확률적이며 학습 중에 서로 다른 토큰화를 샘플링할 수 있습니다.</p> <!--[1--><h2 class="relative group"><a id="sentencepiece" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#sentencepiece"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->SentencePiece<!----></span></h2><!--]--><!----> <p><a href="https://huggingface.co/papers/1808.06226" rel="nofollow">SentencePiece</a>는 원시 텍스트에 BPE 또는 Unigram을 직접 적용하는 토큰화 라이브러리입니다. 표준 BPE와 Unigram은 공백이 단어를 구분한다고 가정하지만, 이는 공백을 사용하지 않는 중국어와 일본어 같은 언어에는 적용되지 않습니다.</p> <ol><li>SentencePiece는 입력 텍스트를 원시 바이트 또는 문자 스트림으로 취급하며, <code>"▁"</code>로 표현되는 공백 문자를 어휘에 포함합니다.</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->(&quot;▁hug&quot;, 10), (&quot;▁pug&quot;, 5), (&quot;▁pun&quot;, 12), (&quot;▁bun&quot;, 4), (&quot;▁hugs&quot;, 5)<!----></pre></div><!----> <ol start="2"><li>SentencePiece는 그런 다음 텍스트에 BPE 또는 Unigram을 적용합니다.</li></ol> <p>디코딩 시 SentencePiece는 모든 토큰을 연결하고 <code>"▁"</code>를 공백으로 대체합니다.</p> <!--[1--><h2 class="relative group"><a id="wordpiece" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#wordpiece"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->WordPiece<!----></span></h2><!--]--><!----> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/qpv6ms_t_1A" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p><a href="https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/37842.pdf" rel="nofollow">WordPiece</a>는 <a href="./model_doc/distilbert">DistilBERT</a>와 <a href="./model_doc/electra">Electra</a>와 같은 BERT 계열 모델에 사용되는 토큰화 알고리즘입니다.</p> <p><a href="#byte-pair-encoding-bpe">BPE</a>와 유사하며, 하단에서부터 쌍을 반복적으로 병합하지만, 쌍을 선택하는 방식이 다릅니다.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->(&quot;h&quot; &quot;u&quot; &quot;g&quot;, 10), (&quot;p&quot; &quot;u&quot; &quot;g&quot;, 5), (&quot;p&quot; &quot;u&quot; &quot;n&quot;, 12), (&quot;b&quot; &quot;u&quot; &quot;n&quot;, 4), (&quot;h&quot; &quot;u&quot; &quot;g&quot; &quot;s&quot;, 5)<!----></pre></div><!----> <p>WordPiece은 학습 데이터의 확률을 최대화하는 쌍을 병합합니다.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->score(&quot;u&quot;, &quot;g&quot;) = frequency(&quot;ug&quot;) / (frequency(&quot;u&quot;) × frequency(&quot;g&quot;))<!----></pre></div><!----> <table><thead><tr><th>pair</th><th>주파수</th><th>score</th></tr></thead><tbody><tr><td><code>"u"</code> + <code>"g"</code></td><td>20</td><td>20 / (36 × 20) = 0.028</td></tr><tr><td><code>"u"</code> + <code>"n"</code></td><td>16</td><td>16 / (36 × 16) = 0.028</td></tr><tr><td><code>"h"</code> + <code>"u"</code></td><td>15</td><td>15 / (15 × 36) = 0.028</td></tr><tr><td><code>"g"</code> + <code>"s"</code></td><td>5</td><td>5 / (20 × 5) = 0.050</td></tr></tbody></table> <p><code>"g"</code>와 <code>"s"</code>를 병합하는 쪽이 점수가 높습니다. 이때 병합된 토큰의 출현 빈도가 개별 토큰의 빈도에서 기대되는 값보다 높을 때입니다. BPE는 단순히 가장 많이 나타나는 쌍을 병합합니다. WordPiece는 각 병합이 얼마나 <em>정보량이 많은지</em>를 측정합니다. 우연히 함께 나타날 확률보다 훨씬 더 자주 함께 나타나는 두 토큰이 먼저 병합됩니다.</p> <!--[1--><h2 class="relative group"><a id="word-level-tokenization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#word-level-tokenization"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->단어 수준 토큰화<!----></span></h2><!--]--><!----> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/nhJxYji1aho" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen=""></iframe><!----> <p>Word-level tokenization은 공백, 구두점 또는 언어별 규칙에 따라 텍스트를 토큰으로 분할합니다.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->[&quot;Do&quot;, &quot;n&#x27;t&quot;, &quot;you&quot;, &quot;love&quot;, &quot;🤗&quot;, &quot;Transformers&quot;, &quot;?&quot;, &quot;We&quot;, &quot;sure&quot;, &quot;do&quot;, &quot;.&quot;]<!----></pre></div><!----> <p>모든 고유한 단어와 그 모든 변형(<code>"love"</code>, <code>"loving"</code>, <code>"loved"</code>, <code>"lovingly"</code>)에 각각 고유한 토큰이 필요하기 때문에 어휘 크기가 매우 커집니다. 이로 인해 임베딩 행렬이 방대해져 메모리와 계산량이 증가합니다. 어휘에 없는 단어는 <code>"&lt;unk>"</code> 토큰으로 매핑되므로 모델이 새로운 단어를 처리할 수 없습니다.</p> <!--[1--><h2 class="relative group"><a id="character-level-tokenization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#character-level-tokenization"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->문자 단위 토큰화<!----></span></h2><!--]--><!----> <p>Character-level tokenization은 텍스트를 개별 문자로 분할합니다.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 복사됨</div><!----></button><!----></div> <pre class="language-text "><!---->[&quot;D&quot;, &quot;o&quot;, &quot;n&quot;, &quot;&#x27;&quot;, &quot;t&quot;, &quot;y&quot;, &quot;o&quot;, &quot;u&quot;, &quot;l&quot;, &quot;o&quot;, &quot;v&quot;, &quot;e&quot;]<!----></pre></div><!----> <p>어휘가 작고 모든 단어를 표현할 수 있으므로 <code>"&lt;unk>"</code> 문제가 없습니다. 하지만 시퀀스가 훨씬 길어집니다. <code>"l"</code>과 같은 문자는 <code>"love"</code>보다 훨씬 적은 의미를 담고 있어 성능이 저하됩니다.</p> <!--[1--><h2 class="relative group"><a id="resources" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resources"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->리소스<!----></span></h2><!--]--><!----> <ul><li>LLM 과정의 <a href="https://huggingface.co/learn/llm-course/chapter6/1" rel="nofollow">6장</a>에서는 토크나이저를 처음부터 학습하는 방법과 BPE, Unigram, WordPiece 알고리즘 간의 차이를 설명합니다.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/tokenizer_summary.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1y3v6y9 = {
						base: "/docs/transformers/main/en",
						assets: "/docs/transformers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/transformers/main/en/_app/immutable/entry/start.ByZEd4qm.js"),
						import("/docs/transformers/main/en/_app/immutable/entry/app.CQqZS61p.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 766],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 43.7 kB
Xet hash:: ed1a5bba6cdd957c1b17b89c1685687b8a09d642808341f23f8177681e543007
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.