Buckets:
| import"../chunks/DsnmJJEf.js";import{i as I,h as k,C,H as t,T as G,a as e,E as X,s as Z}from"../chunks/DQ-9pJ_R.js";import{p as z,o as R,f as a,g as B,i as n,j as V,k as o,n as v}from"../chunks/CzLolCd2.js";const _='{"title":"Training from memory","local":"training-from-memory","sections":[{"title":"The most basic way","local":"the-most-basic-way","sections":[],"depth":2},{"title":"Using the 🤗 Datasets library","local":"using-the--datasets-library","sections":[],"depth":2},{"title":"Using gzip files","local":"using-gzip-files","sections":[],"depth":2}],"depth":1}';var W=o('<meta name="hf:doc:metadata"/>'),Q=o(`<p>Not supported by the rc0 bindings yet. rc0 loads a <code>tokenizer.json</code> and encodes and | |
| decodes with it — building a tokenizer from its components, editing one, saving one | |
| and training are not exposed. They are coming soon, along with the other bindings. | |
| See <a href="https://github.com/huggingface/tokenizers/blob/main/REQUIRED_FOR_V1.md" rel="nofollow"><code>REQUIRED_FOR_V1.md</code></a> for the full list.</p>`),A=o(`<p></p> <!> <!> <!> <p>In the <a href="quicktour">Quicktour</a>, we saw how to build and train a | |
| tokenizer using text files, but we can actually use any Python Iterator. | |
| In this section we’ll see a few different ways of training our | |
| tokenizer.</p> <p>For all the examples listed below, we’ll use the same <code>Tokenizer</code> and <code>~tokenizers.trainers.Trainer</code>, built as | |
| following:</p> <!> <p>This tokenizer is based on the <code>~tokenizers.models.Unigram</code> model. It | |
| takes care of normalizing the input using the NFKC Unicode normalization | |
| method, and uses a <code>ByteLevel</code> pre-tokenizer with the corresponding decoder.</p> <p>For more information on the components used here, you can check <a href="components">here</a>.</p> <!> <p>As you probably guessed already, the easiest way to train our tokenizer | |
| is by using a <code>List</code>{.interpreted-text role=“obj”}:</p> <!> <p>Easy, right? You can use anything working as an iterator here, be it a <code>List</code>{.interpreted-text role=“obj”}, <code>Tuple</code>{.interpreted-text | |
| role=“obj”}, or a <code>np.Array</code>{.interpreted-text role=“obj”}. Anything | |
| works as long as it provides strings.</p> <!> <p>An awesome way to access one of the many datasets that exist out there | |
| is by using the 🤗 Datasets library. For more information about it, you | |
| should check <a href="https://huggingface.co/docs/datasets/" rel="nofollow">the official documentation | |
| here</a>.</p> <p>Let’s start by loading our dataset:</p> <!> <p>The next step is to build an iterator over this dataset. The easiest way | |
| to do this is probably by using a generator:</p> <!> <p>As you can see here, for improved efficiency we can actually provide a | |
| batch of examples used to train, instead of iterating over them one by | |
| one. By doing so, we can expect performances very similar to those we | |
| got while training directly from files.</p> <p>With our iterator ready, we just need to launch the training. In order | |
| to improve the look of our progress bars, we can specify the total | |
| length of the dataset:</p> <!> <p>And that’s it!</p> <!> <p>Since gzip files in Python can be used as iterators, it is extremely | |
| simple to train on such files:</p> <!> <p>Now if we wanted to train from multiple gzip files, it wouldn’t be much | |
| harder:</p> <!> <p>And voilà!</p> <!> <p></p>`,1);function S(b,g){z(g,!1),R(()=>{new URLSearchParams(window.location.search).get("fw")}),I();var i=A();k("1x14sps",s=>{var l=W();Z(l,"content",_),n(s,l)});var r=a(B(i),2);C(r,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var p=a(r,2);t(p,{title:"Training from memory",local:"training-from-memory",headingTag:"h1"});var y=a(p,2);G(y,{warning:!0,children:(s,l)=>{var f=Q();n(s,f)},$$slots:{default:!0}});var d=a(y,6);e(d,{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMkMlMjBkZWNvZGVycyUyQyUyMG1vZGVscyUyQyUyMG5vcm1hbGl6ZXJzJTJDJTIwcHJlX3Rva2VuaXplcnMlMkMlMjB0cmFpbmVycyUwQXRva2VuaXplciUyMCUzRCUyMFRva2VuaXplcihtb2RlbHMuVW5pZ3JhbSgpKSUwQXRva2VuaXplci5ub3JtYWxpemVyJTIwJTNEJTIwbm9ybWFsaXplcnMuTkZLQygpJTBBdG9rZW5pemVyLnByZV90b2tlbml6ZXIlMjAlM0QlMjBwcmVfdG9rZW5pemVycy5CeXRlTGV2ZWwoKSUwQXRva2VuaXplci5kZWNvZGVyJTIwJTNEJTIwZGVjb2RlcnMuQnl0ZUxldmVsKCklMEF0cmFpbmVyJTIwJTNEJTIwdHJhaW5lcnMuVW5pZ3JhbVRyYWluZXIoJTBBJTIwJTIwJTIwJTIwdm9jYWJfc2l6ZSUzRDIwMDAwJTJDJTBBJTIwJTIwJTIwJTIwaW5pdGlhbF9hbHBoYWJldCUzRHByZV90b2tlbml6ZXJzLkJ5dGVMZXZlbC5hbHBoYWJldCgpJTJDJTBBJTIwJTIwJTIwJTIwc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlM0NQQUQlM0UlMjIlMkMlMjAlMjIlM0NCT1MlM0UlMjIlMkMlMjAlMjIlM0NFT1MlM0UlMjIlNUQlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer, decoders, models, normalizers, pre_tokenizers, trainers | |
| tokenizer = Tokenizer(models.Unigram()) | |
| tokenizer.normalizer = normalizers.NFKC() | |
| tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel() | |
| tokenizer.decoder = decoders.ByteLevel() | |
| trainer = trainers.UnigramTrainer( | |
| vocab_size=<span class="hljs-number">20000</span>, | |
| initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), | |
| special_tokens=[<span class="hljs-string">"<PAD>"</span>, <span class="hljs-string">"<BOS>"</span>, <span class="hljs-string">"<EOS>"</span>], | |
| )`,lang:"python",wrap:!1});var M=a(d,6);t(M,{title:"The most basic way",local:"the-most-basic-way",headingTag:"h2"});var c=a(M,4);e(c,{code:"JTIzJTIwRmlyc3QlMjBmZXclMjBsaW5lcyUyMG9mJTIwdGhlJTIwJTIyWmVuJTIwb2YlMjBQeXRob24lMjIlMjBodHRwcyUzQSUyRiUyRnd3dy5weXRob24ub3JnJTJGZGV2JTJGcGVwcyUyRnBlcC0wMDIwJTJGJTBBZGF0YSUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMkJlYXV0aWZ1bCUyMGlzJTIwYmV0dGVyJTIwdGhhbiUyMHVnbHkuJTIyJTBBJTIwJTIwJTIwJTIwJTIyRXhwbGljaXQlMjBpcyUyMGJldHRlciUyMHRoYW4lMjBpbXBsaWNpdC4lMjIlMEElMjAlMjAlMjAlMjAlMjJTaW1wbGUlMjBpcyUyMGJldHRlciUyMHRoYW4lMjBjb21wbGV4LiUyMiUwQSUyMCUyMCUyMCUyMCUyMkNvbXBsZXglMjBpcyUyMGJldHRlciUyMHRoYW4lMjBjb21wbGljYXRlZC4lMjIlMEElMjAlMjAlMjAlMjAlMjJGbGF0JTIwaXMlMjBiZXR0ZXIlMjB0aGFuJTIwbmVzdGVkLiUyMiUwQSUyMCUyMCUyMCUyMCUyMlNwYXJzZSUyMGlzJTIwYmV0dGVyJTIwdGhhbiUyMGRlbnNlLiUyMiUwQSUyMCUyMCUyMCUyMCUyMlJlYWRhYmlsaXR5JTIwY291bnRzLiUyMiUwQSU1RCUwQXRva2VuaXplci50cmFpbl9mcm9tX2l0ZXJhdG9yKGRhdGElMkMlMjB0cmFpbmVyJTNEdHJhaW5lcik=",highlighted:`<span class="hljs-comment"># First few lines of the "Zen of Python" https://www.python.org/dev/peps/pep-0020/</span> | |
| data = [ | |
| <span class="hljs-string">"Beautiful is better than ugly."</span> | |
| <span class="hljs-string">"Explicit is better than implicit."</span> | |
| <span class="hljs-string">"Simple is better than complex."</span> | |
| <span class="hljs-string">"Complex is better than complicated."</span> | |
| <span class="hljs-string">"Flat is better than nested."</span> | |
| <span class="hljs-string">"Sparse is better than dense."</span> | |
| <span class="hljs-string">"Readability counts."</span> | |
| ] | |
| tokenizer.train_from_iterator(data, trainer=trainer)`,lang:"python",wrap:!1});var h=a(c,4);t(h,{title:"Using the 🤗 Datasets library",local:"using-the--datasets-library",headingTag:"h2"});var m=a(h,6);e(m,{code:"aW1wb3J0JTIwZGF0YXNldHMlMjAlMjAlMjMlMjB0eXBlJTNBJTIwaWdub3JlJTVCaW1wb3J0LW5vdC1mb3VuZCU1RCUwQWRhdGFzZXQlMjAlM0QlMjBkYXRhc2V0cy5sb2FkX2RhdGFzZXQoJTIyU2FsZXNmb3JjZSUyRndpa2l0ZXh0JTIyJTJDJTIwJTIyd2lraXRleHQtMTAzLXJhdy12MSUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMkJ0ZXN0JTJCdmFsaWRhdGlvbiUyMik=",highlighted:`<span class="hljs-keyword">import</span> datasets <span class="hljs-comment"># type: ignore[<span class="hljs-keyword">import</span>-<span class="hljs-keyword">not</span>-found]</span> | |
| dataset = datasets.load_dataset(<span class="hljs-string">"Salesforce/wikitext"</span>, <span class="hljs-string">"wikitext-103-raw-v1"</span>, split=<span class="hljs-string">"train+test+validation"</span>)`,lang:"python",wrap:!1});var w=a(m,4);e(w,{code:"ZGVmJTIwYmF0Y2hfaXRlcmF0b3IoYmF0Y2hfc2l6ZSUzRDEwMDApJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwT25seSUyMGtlZXAlMjB0aGUlMjB0ZXh0JTIwY29sdW1uJTIwdG8lMjBhdm9pZCUyMGRlY29kaW5nJTIwdGhlJTIwcmVzdCUyMG9mJTIwdGhlJTIwY29sdW1ucyUyMHVubmVjZXNzYXJpbHklMEElMjAlMjAlMjAlMjB0b2tfZGF0YXNldCUyMCUzRCUyMGRhdGFzZXQuc2VsZWN0X2NvbHVtbnMoJTIydGV4dCUyMiklMEElMjAlMjAlMjAlMjBmb3IlMjBiYXRjaCUyMGluJTIwdG9rX2RhdGFzZXQuaXRlcihiYXRjaF9zaXplKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHlpZWxkJTIwYmF0Y2glNUIlMjJ0ZXh0JTIyJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">batch_iterator</span>(<span class="hljs-params">batch_size=<span class="hljs-number">1000</span></span>): | |
| <span class="hljs-comment"># Only keep the text column to avoid decoding the rest of the columns unnecessarily</span> | |
| tok_dataset = dataset.select_columns(<span class="hljs-string">"text"</span>) | |
| <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> tok_dataset.<span class="hljs-built_in">iter</span>(batch_size): | |
| <span class="hljs-keyword">yield</span> batch[<span class="hljs-string">"text"</span>]`,lang:"python",wrap:!1});var U=a(w,6);e(U,{code:"dG9rZW5pemVyLnRyYWluX2Zyb21faXRlcmF0b3IoYmF0Y2hfaXRlcmF0b3IoKSUyQyUyMHRyYWluZXIlM0R0cmFpbmVyJTJDJTIwbGVuZ3RoJTNEbGVuKGRhdGFzZXQpKQ==",highlighted:'tokenizer.train_from_iterator(batch_iterator(), trainer=trainer, length=<span class="hljs-built_in">len</span>(dataset))',lang:"python",wrap:!1});var u=a(U,4);t(u,{title:"Using gzip files",local:"using-gzip-files",headingTag:"h2"});var j=a(u,4);e(j,{code:"aW1wb3J0JTIwZ3ppcCUwQXdpdGglMjBnemlwLm9wZW4oJTIyZGF0YSUyRm15LWZpbGUuMC5neiUyMiUyQyUyMCUyMnJ0JTIyKSUyMGFzJTIwZiUzQSUwQSUyMCUyMCUyMCUyMHRva2VuaXplci50cmFpbl9mcm9tX2l0ZXJhdG9yKGYlMkMlMjB0cmFpbmVyJTNEdHJhaW5lcik=",highlighted:`<span class="hljs-keyword">import</span> gzip | |
| <span class="hljs-keyword">with</span> gzip.<span class="hljs-built_in">open</span>(<span class="hljs-string">"data/my-file.0.gz"</span>, <span class="hljs-string">"rt"</span>) <span class="hljs-keyword">as</span> f: | |
| tokenizer.train_from_iterator(f, trainer=trainer)`,lang:"python",wrap:!1});var J=a(j,4);e(J,{code:"ZmlsZXMlMjAlM0QlMjAlNUIlMjJkYXRhJTJGbXktZmlsZS4wLmd6JTIyJTJDJTIwJTIyZGF0YSUyRm15LWZpbGUuMS5neiUyMiUyQyUyMCUyMmRhdGElMkZteS1maWxlLjIuZ3olMjIlNUQlMEFkZWYlMjBnemlwX2l0ZXJhdG9yKCklM0ElMEElMjAlMjAlMjAlMjBmb3IlMjBwYXRoJTIwaW4lMjBmaWxlcyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdpdGglMjBnemlwLm9wZW4ocGF0aCUyQyUyMCUyMnJ0JTIyKSUyMGFzJTIwZiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGxpbmUlMjBpbiUyMGYlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB5aWVsZCUyMGxpbmUlMEF0b2tlbml6ZXIudHJhaW5fZnJvbV9pdGVyYXRvcihnemlwX2l0ZXJhdG9yKCklMkMlMjB0cmFpbmVyJTNEdHJhaW5lcik=",highlighted:`files = [<span class="hljs-string">"data/my-file.0.gz"</span>, <span class="hljs-string">"data/my-file.1.gz"</span>, <span class="hljs-string">"data/my-file.2.gz"</span>] | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">gzip_iterator</span>(): | |
| <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> files: | |
| <span class="hljs-keyword">with</span> gzip.<span class="hljs-built_in">open</span>(path, <span class="hljs-string">"rt"</span>) <span class="hljs-keyword">as</span> f: | |
| <span class="hljs-keyword">for</span> line <span class="hljs-keyword">in</span> f: | |
| <span class="hljs-keyword">yield</span> line | |
| tokenizer.train_from_iterator(gzip_iterator(), trainer=trainer)`,lang:"python",wrap:!1});var T=a(J,4);X(T,{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/training_from_memory.mdx"}),v(2),n(b,i),V()}export{S as component}; | |
Xet Storage Details
- Size:
- 11.8 kB
- Xet hash:
- cf0ea26d20a8adb7603c7f0b9c8920455fbb3f237ad50338b1832268156a396e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.