Buckets:

download
raw
11.8 kB
import"../chunks/DsnmJJEf.js";import{i as I,h as k,C,H as t,T as G,a as e,E as X,s as Z}from"../chunks/DQ-9pJ_R.js";import{p as z,o as R,f as a,g as B,i as n,j as V,k as o,n as v}from"../chunks/CzLolCd2.js";const _='{"title":"Training from memory","local":"training-from-memory","sections":[{"title":"The most basic way","local":"the-most-basic-way","sections":[],"depth":2},{"title":"Using the 🤗 Datasets library","local":"using-the--datasets-library","sections":[],"depth":2},{"title":"Using gzip files","local":"using-gzip-files","sections":[],"depth":2}],"depth":1}';var W=o('<meta name="hf:doc:metadata"/>'),Q=o(`<p>Not supported by the rc0 bindings yet. rc0 loads a <code>tokenizer.json</code> and encodes and
decodes with it — building a tokenizer from its components, editing one, saving one
and training are not exposed. They are coming soon, along with the other bindings.
See <a href="https://github.com/huggingface/tokenizers/blob/main/REQUIRED_FOR_V1.md" rel="nofollow"><code>REQUIRED_FOR_V1.md</code></a> for the full list.</p>`),A=o(`<p></p> <!> <!> <!> <p>In the <a href="quicktour">Quicktour</a>, we saw how to build and train a
tokenizer using text files, but we can actually use any Python Iterator.
In this section we’ll see a few different ways of training our
tokenizer.</p> <p>For all the examples listed below, we’ll use the same <code>Tokenizer</code> and <code>~tokenizers.trainers.Trainer</code>, built as
following:</p> <!> <p>This tokenizer is based on the <code>~tokenizers.models.Unigram</code> model. It
takes care of normalizing the input using the NFKC Unicode normalization
method, and uses a <code>ByteLevel</code> pre-tokenizer with the corresponding decoder.</p> <p>For more information on the components used here, you can check <a href="components">here</a>.</p> <!> <p>As you probably guessed already, the easiest way to train our tokenizer
is by using a <code>List</code>&#123;.interpreted-text role=“obj”}:</p> <!> <p>Easy, right? You can use anything working as an iterator here, be it a <code>List</code>&#123;.interpreted-text role=“obj”}, <code>Tuple</code>&#123;.interpreted-text
role=“obj”}, or a <code>np.Array</code>&#123;.interpreted-text role=“obj”}. Anything
works as long as it provides strings.</p> <!> <p>An awesome way to access one of the many datasets that exist out there
is by using the 🤗 Datasets library. For more information about it, you
should check <a href="https://huggingface.co/docs/datasets/" rel="nofollow">the official documentation
here</a>.</p> <p>Let’s start by loading our dataset:</p> <!> <p>The next step is to build an iterator over this dataset. The easiest way
to do this is probably by using a generator:</p> <!> <p>As you can see here, for improved efficiency we can actually provide a
batch of examples used to train, instead of iterating over them one by
one. By doing so, we can expect performances very similar to those we
got while training directly from files.</p> <p>With our iterator ready, we just need to launch the training. In order
to improve the look of our progress bars, we can specify the total
length of the dataset:</p> <!> <p>And that’s it!</p> <!> <p>Since gzip files in Python can be used as iterators, it is extremely
simple to train on such files:</p> <!> <p>Now if we wanted to train from multiple gzip files, it wouldn’t be much
harder:</p> <!> <p>And voilà!</p> <!> <p></p>`,1);function S(b,g){z(g,!1),R(()=>{new URLSearchParams(window.location.search).get("fw")}),I();var i=A();k("1x14sps",s=>{var l=W();Z(l,"content",_),n(s,l)});var r=a(B(i),2);C(r,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var p=a(r,2);t(p,{title:"Training from memory",local:"training-from-memory",headingTag:"h1"});var y=a(p,2);G(y,{warning:!0,children:(s,l)=>{var f=Q();n(s,f)},$$slots:{default:!0}});var d=a(y,6);e(d,{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMkMlMjBkZWNvZGVycyUyQyUyMG1vZGVscyUyQyUyMG5vcm1hbGl6ZXJzJTJDJTIwcHJlX3Rva2VuaXplcnMlMkMlMjB0cmFpbmVycyUwQXRva2VuaXplciUyMCUzRCUyMFRva2VuaXplcihtb2RlbHMuVW5pZ3JhbSgpKSUwQXRva2VuaXplci5ub3JtYWxpemVyJTIwJTNEJTIwbm9ybWFsaXplcnMuTkZLQygpJTBBdG9rZW5pemVyLnByZV90b2tlbml6ZXIlMjAlM0QlMjBwcmVfdG9rZW5pemVycy5CeXRlTGV2ZWwoKSUwQXRva2VuaXplci5kZWNvZGVyJTIwJTNEJTIwZGVjb2RlcnMuQnl0ZUxldmVsKCklMEF0cmFpbmVyJTIwJTNEJTIwdHJhaW5lcnMuVW5pZ3JhbVRyYWluZXIoJTBBJTIwJTIwJTIwJTIwdm9jYWJfc2l6ZSUzRDIwMDAwJTJDJTBBJTIwJTIwJTIwJTIwaW5pdGlhbF9hbHBoYWJldCUzRHByZV90b2tlbml6ZXJzLkJ5dGVMZXZlbC5hbHBoYWJldCgpJTJDJTBBJTIwJTIwJTIwJTIwc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlM0NQQUQlM0UlMjIlMkMlMjAlMjIlM0NCT1MlM0UlMjIlMkMlMjAlMjIlM0NFT1MlM0UlMjIlNUQlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer, decoders, models, normalizers, pre_tokenizers, trainers
tokenizer = Tokenizer(models.Unigram())
tokenizer.normalizer = normalizers.NFKC()
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
tokenizer.decoder = decoders.ByteLevel()
trainer = trainers.UnigramTrainer(
vocab_size=<span class="hljs-number">20000</span>,
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
special_tokens=[<span class="hljs-string">&quot;&lt;PAD&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;BOS&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;EOS&gt;&quot;</span>],
)`,lang:"python",wrap:!1});var M=a(d,6);t(M,{title:"The most basic way",local:"the-most-basic-way",headingTag:"h2"});var c=a(M,4);e(c,{code:"JTIzJTIwRmlyc3QlMjBmZXclMjBsaW5lcyUyMG9mJTIwdGhlJTIwJTIyWmVuJTIwb2YlMjBQeXRob24lMjIlMjBodHRwcyUzQSUyRiUyRnd3dy5weXRob24ub3JnJTJGZGV2JTJGcGVwcyUyRnBlcC0wMDIwJTJGJTBBZGF0YSUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMkJlYXV0aWZ1bCUyMGlzJTIwYmV0dGVyJTIwdGhhbiUyMHVnbHkuJTIyJTBBJTIwJTIwJTIwJTIwJTIyRXhwbGljaXQlMjBpcyUyMGJldHRlciUyMHRoYW4lMjBpbXBsaWNpdC4lMjIlMEElMjAlMjAlMjAlMjAlMjJTaW1wbGUlMjBpcyUyMGJldHRlciUyMHRoYW4lMjBjb21wbGV4LiUyMiUwQSUyMCUyMCUyMCUyMCUyMkNvbXBsZXglMjBpcyUyMGJldHRlciUyMHRoYW4lMjBjb21wbGljYXRlZC4lMjIlMEElMjAlMjAlMjAlMjAlMjJGbGF0JTIwaXMlMjBiZXR0ZXIlMjB0aGFuJTIwbmVzdGVkLiUyMiUwQSUyMCUyMCUyMCUyMCUyMlNwYXJzZSUyMGlzJTIwYmV0dGVyJTIwdGhhbiUyMGRlbnNlLiUyMiUwQSUyMCUyMCUyMCUyMCUyMlJlYWRhYmlsaXR5JTIwY291bnRzLiUyMiUwQSU1RCUwQXRva2VuaXplci50cmFpbl9mcm9tX2l0ZXJhdG9yKGRhdGElMkMlMjB0cmFpbmVyJTNEdHJhaW5lcik=",highlighted:`<span class="hljs-comment"># First few lines of the &quot;Zen of Python&quot; https://www.python.org/dev/peps/pep-0020/</span>
data = [
<span class="hljs-string">&quot;Beautiful is better than ugly.&quot;</span>
<span class="hljs-string">&quot;Explicit is better than implicit.&quot;</span>
<span class="hljs-string">&quot;Simple is better than complex.&quot;</span>
<span class="hljs-string">&quot;Complex is better than complicated.&quot;</span>
<span class="hljs-string">&quot;Flat is better than nested.&quot;</span>
<span class="hljs-string">&quot;Sparse is better than dense.&quot;</span>
<span class="hljs-string">&quot;Readability counts.&quot;</span>
]
tokenizer.train_from_iterator(data, trainer=trainer)`,lang:"python",wrap:!1});var h=a(c,4);t(h,{title:"Using the 🤗 Datasets library",local:"using-the--datasets-library",headingTag:"h2"});var m=a(h,6);e(m,{code:"aW1wb3J0JTIwZGF0YXNldHMlMjAlMjAlMjMlMjB0eXBlJTNBJTIwaWdub3JlJTVCaW1wb3J0LW5vdC1mb3VuZCU1RCUwQWRhdGFzZXQlMjAlM0QlMjBkYXRhc2V0cy5sb2FkX2RhdGFzZXQoJTIyU2FsZXNmb3JjZSUyRndpa2l0ZXh0JTIyJTJDJTIwJTIyd2lraXRleHQtMTAzLXJhdy12MSUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMkJ0ZXN0JTJCdmFsaWRhdGlvbiUyMik=",highlighted:`<span class="hljs-keyword">import</span> datasets <span class="hljs-comment"># type: ignore[<span class="hljs-keyword">import</span>-<span class="hljs-keyword">not</span>-found]</span>
dataset = datasets.load_dataset(<span class="hljs-string">&quot;Salesforce/wikitext&quot;</span>, <span class="hljs-string">&quot;wikitext-103-raw-v1&quot;</span>, split=<span class="hljs-string">&quot;train+test+validation&quot;</span>)`,lang:"python",wrap:!1});var w=a(m,4);e(w,{code:"ZGVmJTIwYmF0Y2hfaXRlcmF0b3IoYmF0Y2hfc2l6ZSUzRDEwMDApJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwT25seSUyMGtlZXAlMjB0aGUlMjB0ZXh0JTIwY29sdW1uJTIwdG8lMjBhdm9pZCUyMGRlY29kaW5nJTIwdGhlJTIwcmVzdCUyMG9mJTIwdGhlJTIwY29sdW1ucyUyMHVubmVjZXNzYXJpbHklMEElMjAlMjAlMjAlMjB0b2tfZGF0YXNldCUyMCUzRCUyMGRhdGFzZXQuc2VsZWN0X2NvbHVtbnMoJTIydGV4dCUyMiklMEElMjAlMjAlMjAlMjBmb3IlMjBiYXRjaCUyMGluJTIwdG9rX2RhdGFzZXQuaXRlcihiYXRjaF9zaXplKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHlpZWxkJTIwYmF0Y2glNUIlMjJ0ZXh0JTIyJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">batch_iterator</span>(<span class="hljs-params">batch_size=<span class="hljs-number">1000</span></span>):
<span class="hljs-comment"># Only keep the text column to avoid decoding the rest of the columns unnecessarily</span>
tok_dataset = dataset.select_columns(<span class="hljs-string">&quot;text&quot;</span>)
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> tok_dataset.<span class="hljs-built_in">iter</span>(batch_size):
<span class="hljs-keyword">yield</span> batch[<span class="hljs-string">&quot;text&quot;</span>]`,lang:"python",wrap:!1});var U=a(w,6);e(U,{code:"dG9rZW5pemVyLnRyYWluX2Zyb21faXRlcmF0b3IoYmF0Y2hfaXRlcmF0b3IoKSUyQyUyMHRyYWluZXIlM0R0cmFpbmVyJTJDJTIwbGVuZ3RoJTNEbGVuKGRhdGFzZXQpKQ==",highlighted:'tokenizer.train_from_iterator(batch_iterator(), trainer=trainer, length=<span class="hljs-built_in">len</span>(dataset))',lang:"python",wrap:!1});var u=a(U,4);t(u,{title:"Using gzip files",local:"using-gzip-files",headingTag:"h2"});var j=a(u,4);e(j,{code:"aW1wb3J0JTIwZ3ppcCUwQXdpdGglMjBnemlwLm9wZW4oJTIyZGF0YSUyRm15LWZpbGUuMC5neiUyMiUyQyUyMCUyMnJ0JTIyKSUyMGFzJTIwZiUzQSUwQSUyMCUyMCUyMCUyMHRva2VuaXplci50cmFpbl9mcm9tX2l0ZXJhdG9yKGYlMkMlMjB0cmFpbmVyJTNEdHJhaW5lcik=",highlighted:`<span class="hljs-keyword">import</span> gzip
<span class="hljs-keyword">with</span> gzip.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;data/my-file.0.gz&quot;</span>, <span class="hljs-string">&quot;rt&quot;</span>) <span class="hljs-keyword">as</span> f:
tokenizer.train_from_iterator(f, trainer=trainer)`,lang:"python",wrap:!1});var J=a(j,4);e(J,{code:"ZmlsZXMlMjAlM0QlMjAlNUIlMjJkYXRhJTJGbXktZmlsZS4wLmd6JTIyJTJDJTIwJTIyZGF0YSUyRm15LWZpbGUuMS5neiUyMiUyQyUyMCUyMmRhdGElMkZteS1maWxlLjIuZ3olMjIlNUQlMEFkZWYlMjBnemlwX2l0ZXJhdG9yKCklM0ElMEElMjAlMjAlMjAlMjBmb3IlMjBwYXRoJTIwaW4lMjBmaWxlcyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdpdGglMjBnemlwLm9wZW4ocGF0aCUyQyUyMCUyMnJ0JTIyKSUyMGFzJTIwZiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGxpbmUlMjBpbiUyMGYlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB5aWVsZCUyMGxpbmUlMEF0b2tlbml6ZXIudHJhaW5fZnJvbV9pdGVyYXRvcihnemlwX2l0ZXJhdG9yKCklMkMlMjB0cmFpbmVyJTNEdHJhaW5lcik=",highlighted:`files = [<span class="hljs-string">&quot;data/my-file.0.gz&quot;</span>, <span class="hljs-string">&quot;data/my-file.1.gz&quot;</span>, <span class="hljs-string">&quot;data/my-file.2.gz&quot;</span>]
<span class="hljs-keyword">def</span> <span class="hljs-title function_">gzip_iterator</span>():
<span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> files:
<span class="hljs-keyword">with</span> gzip.<span class="hljs-built_in">open</span>(path, <span class="hljs-string">&quot;rt&quot;</span>) <span class="hljs-keyword">as</span> f:
<span class="hljs-keyword">for</span> line <span class="hljs-keyword">in</span> f:
<span class="hljs-keyword">yield</span> line
tokenizer.train_from_iterator(gzip_iterator(), trainer=trainer)`,lang:"python",wrap:!1});var T=a(J,4);X(T,{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/training_from_memory.mdx"}),v(2),n(b,i),V()}export{S as component};

Xet Storage Details

Size:
11.8 kB
·
Xet hash:
cf0ea26d20a8adb7603c7f0b9c8920455fbb3f237ad50338b1832268156a396e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.