Tokenizer CPU

README.md

@@ -53,7 +53,10 @@ The full explanation, the compiler and the tests are on GitHub: https://github.c
 | `4-state-busy-beaver` | the 4-state busy beaver: halts after 107 ticks with 13 ones | 24 |
 | `binary-counter` | counts in binary forever | 7 |
 | `sort` | a block cellular automaton: every two-letter block of the tape is rewritten in one tick, sorting it | 704 |
+| `rule110` | Rule 110, which is Turing complete: every cell updates every two ticks, and the tape grows at both edges | 29 |
+| `rule110-ether` | Rule 110 on its ether, the periodic background its gliders travel on, which the edges keep growing | 146 |
 
 Load a subfolder with `AutoTokenizer.from_pretrained("fMindstormer/tokcpu", subfolder="hello")`. Each one has its
-own `start.txt`. `programs/` holds the Brainfuck sources. The sorter's tape never stops changing (a phase
-mark flips every tick), so stop it when the tape repeats every two ticks.
+own `start.txt`. `programs/` holds the Brainfuck sources. The block automata's tapes never stop changing (a
+phase mark flips every tick): stop the sorter when the tape repeats every two ticks, and Rule 110 whenever you
+like.

rule110-ether/start.txt

@@ -0,0 +1 @@
+①◦··██··██··███···█····█·██████·█···█·███████·█·█·█··██··█⑴

rule110-ether/tokenizer.json

@@ -0,0 +1 @@
+{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":185,"content":"··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":186,"content":"·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":187,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":188,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":189,"content":"██⑵","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":190,"content":"··⑶","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":191,"content":"·█⑷","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":192,"content":"██⑸","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":193,"content":"··⑹","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":194,"content":"██⑺","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":195,"content":"██⑻","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":196,"content":"·█⑼","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":197,"content":"██⑽","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":198,"content":"██⑾","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":199,"content":"██⑿","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":200,"content":"·█⒀","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":201,"content":"██⒁","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":202,"content":"··⑴","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":203,"content":"█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":204,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":205,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":206,"content":"█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":207,"content":"█·⑵","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":208,"content":"█·⑶","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":209,"content":"██⑷","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":210,"content":"█·⑸","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":211,"content":"█·⑹","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":212,"content":"█·⑺","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":213,"content":"█·⑻","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":214,"content":"██⑼","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":215,"content":"██⑽","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":216,"content":"█·⑾","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":217,"content":"██⑿","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":218,"content":"██⒀","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":219,"content":"██⒁","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":220,"content":"█·⑴","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":221,"content":"··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":222,"content":"·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":223,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":224,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":225,"content":"██⑵","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":226,"content":"··⑶","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":227,"content":"·█⑷","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":228,"content":"██⑸","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":229,"content":"··⑹","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":230,"content":"██⑺","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":231,"content":"██⑻","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":232,"content":"·█⑼","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":233,"content":"██⑽","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":234,"content":"██⑾","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":235,"content":"██⑿","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":236,"content":"·█⒀","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":237,"content":"██⒁","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":238,"content":"··⑴","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":239,"content":"█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":240,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":241,"content":"·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":242,"content":"··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":243,"content":"··⑵","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":244,"content":"█·⑶","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":245,"content":"██⑷","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":246,"content":"··⑸","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":247,"content":"█·⑹","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":248,"content":"··⑺","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":249,"content":"··⑻","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":250,"content":"██⑼","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":251,"content":"·█⑽","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":252,"content":"··⑾","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":253,"content":"·█⑿","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":254,"content":"██⒀","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":255,"content":"·█⒁","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":256,"content":"█·⑴","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":257,"content":"ea","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":258,"content":"fb","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":259,"content":"②◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":260,"content":"②◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":261,"content":"②◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":262,"content":"②◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":263,"content":"①","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":264,"content":"③◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":265,"content":"③◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":266,"content":"③◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":267,"content":"③◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":268,"content":"②","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":269,"content":"④◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":270,"content":"④◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":271,"content":"④◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":272,"content":"④◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":273,"content":"③","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":274,"content":"⑤◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":275,"content":"⑤◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":276,"content":"⑤◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":277,"content":"⑤◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":278,"content":"④","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":279,"content":"⑥◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":280,"content":"⑥◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":281,"content":"⑥◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":282,"content":"⑥◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":283,"content":"⑤","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":284,"content":"⑦◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":285,"content":"⑦◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":286,"content":"⑦◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":287,"content":"⑦◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":288,"content":"⑥","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":289,"content":"⑧◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":290,"content":"⑧◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":291,"content":"⑧◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":292,"content":"⑧◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":293,"content":"⑦","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":294,"content":"⑨◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":295,"content":"⑨◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":296,"content":"⑨◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":297,"content":"⑨◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":298,"content":"⑧","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":299,"content":"⑩◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":300,"content":"⑩◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":301,"content":"⑩◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":302,"content":"⑩◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":303,"content":"⑨","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":304,"content":"⑪◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":305,"content":"⑪◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":306,"content":"⑪◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":307,"content":"⑪◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":308,"content":"⑩","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":309,"content":"⑫◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":310,"content":"⑫◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":311,"content":"⑫◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":312,"content":"⑫◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":313,"content":"⑪","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":314,"content":"⑬◦█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":315,"content":"⑬◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":316,"content":"⑬◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":317,"content":"⑬◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":318,"content":"⑫","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":319,"content":"⑭◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":320,"content":"⑭◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":321,"content":"⑭◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":322,"content":"⑭◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":323,"content":"⑬","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":324,"content":"①◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":325,"content":"①◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":326,"content":"①◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":327,"content":"①◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":328,"content":"⑭","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":329,"content":"gc","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":330,"content":"hd","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}],"normalizer":null,"pre_tokenizer":{"type":"Split","pattern":{"Regex":".."},"behavior":"Isolated","invert":false},"post_processor":null,"decoder":{"type":"Sequence","decoders":[{"type":"Replace","pattern":{"Regex":"[-]"},"content":""},{"type":"Fuse"}]},"model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":false,"vocab":{"a":0,"b":1,"c":2,"d":3,"e":4,"f":5,"g":6,"h":7,"·":8,"①":9,"②":10,"③":11,"④":12,"⑤":13,"⑥":14,"⑦":15,"⑧":16,"⑨":17,"⑩":18,"⑪":19,"⑫":20,"⑬":21,"⑭":22,"⑴":23,"⑵":24,"⑶":25,"⑷":26,"⑸":27,"⑹":28,"⑺":29,"⑻":30,"⑼":31,"⑽":32,"⑾":33,"⑿":34,"⒀":35,"⒁":36,"█":37,"◦":38,"ae":185,"af":186,"ag":187,"ah":188,"a⑴":189,"a⑵":190,"a⑶":191,"a⑷":192,"a⑸":193,"a⑹":194,"a⑺":195,"a⑻":196,"a⑼":197,"a⑽":198,"a⑾":199,"a⑿":200,"a⒀":201,"a⒁":202,"be":203,"bf":204,"bg":205,"bh":206,"b⑴":207,"b⑵":208,"b⑶":209,"b⑷":210,"b⑸":211,"b⑹":212,"b⑺":213,"b⑻":214,"b⑼":215,"b⑽":216,"b⑾":217,"b⑿":218,"b⒀":219,"b⒁":220,"ce":221,"cf":222,"cg":223,"ch":224,"c⑴":225,"c⑵":226,"c⑶":227,"c⑷":228,"c⑸":229,"c⑹":230,"c⑺":231,"c⑻":232,"c⑼":233,"c⑽":234,"c⑾":235,"c⑿":236,"c⒀":237,"c⒁":238,"de":239,"df":240,"dg":241,"dh":242,"d⑴":243,"d⑵":244,"d⑶":245,"d⑷":246,"d⑸":247,"d⑹":248,"d⑺":249,"d⑻":250,"d⑼":251,"d⑽":252,"d⑾":253,"d⑿":254,"d⒀":255,"d⒁":256,"··":257,"·█":258,"①e":259,"①f":260,"①g":261,"①h":262,"①◦":263,"②e":264,"②f":265,"②g":266,"②h":267,"②◦":268,"③e":269,"③f":270,"③g":271,"③h":272,"③◦":273,"④e":274,"④f":275,"④g":276,"④h":277,"④◦":278,"⑤e":279,"⑤f":280,"⑤g":281,"⑤h":282,"⑤◦":283,"⑥e":284,"⑥f":285,"⑥g":286,"⑥h":287,"⑥◦":288,"⑦e":289,"⑦f":290,"⑦g":291,"⑦h":292,"⑦◦":293,"⑧e":294,"⑧f":295,"⑧g":296,"⑧h":297,"⑧◦":298,"⑨e":299,"⑨f":300,"⑨g":301,"⑨h":302,"⑨◦":303,"⑩e":304,"⑩f":305,"⑩g":306,"⑩h":307,"⑩◦":308,"⑪e":309,"⑪f":310,"⑪g":311,"⑪h":312,"⑪◦":313,"⑫e":314,"⑫f":315,"⑫g":316,"⑫h":317,"⑫◦":318,"⑬e":319,"⑬f":320,"⑬g":321,"⑬h":322,"⑬◦":323,"⑭e":324,"⑭f":325,"⑭g":326,"⑭h":327,"⑭◦":328,"█·":329,"██":330},"merges":[["a","e"],["a","f"],["a","g"],["a","h"],["a","⑴"],["a","⑵"],["a","⑶"],["a","⑷"],["a","⑸"],["a","⑹"],["a","⑺"],["a","⑻"],["a","⑼"],["a","⑽"],["a","⑾"],["a","⑿"],["a","⒀"],["a","⒁"],["b","e"],["b","f"],["b","g"],["b","h"],["b","⑴"],["b","⑵"],["b","⑶"],["b","⑷"],["b","⑸"],["b","⑹"],["b","⑺"],["b","⑻"],["b","⑼"],["b","⑽"],["b","⑾"],["b","⑿"],["b","⒀"],["b","⒁"],["c","e"],["c","f"],["c","g"],["c","h"],["c","⑴"],["c","⑵"],["c","⑶"],["c","⑷"],["c","⑸"],["c","⑹"],["c","⑺"],["c","⑻"],["c","⑼"],["c","⑽"],["c","⑾"],["c","⑿"],["c","⒀"],["c","⒁"],["d","e"],["d","f"],["d","g"],["d","h"],["d","⑴"],["d","⑵"],["d","⑶"],["d","⑷"],["d","⑸"],["d","⑹"],["d","⑺"],["d","⑻"],["d","⑼"],["d","⑽"],["d","⑾"],["d","⑿"],["d","⒀"],["d","⒁"],["·","·"],["·","█"],["①","e"],["①","f"],["①","g"],["①","h"],["①","◦"],["②","e"],["②","f"],["②","g"],["②","h"],["②","◦"],["③","e"],["③","f"],["③","g"],["③","h"],["③","◦"],["④","e"],["④","f"],["④","g"],["④","h"],["④","◦"],["⑤","e"],["⑤","f"],["⑤","g"],["⑤","h"],["⑤","◦"],["⑥","e"],["⑥","f"],["⑥","g"],["⑥","h"],["⑥","◦"],["⑦","e"],["⑦","f"],["⑦","g"],["⑦","h"],["⑦","◦"],["⑧","e"],["⑧","f"],["⑧","g"],["⑧","h"],["⑧","◦"],["⑨","e"],["⑨","f"],["⑨","g"],["⑨","h"],["⑨","◦"],["⑩","e"],["⑩","f"],["⑩","g"],["⑩","h"],["⑩","◦"],["⑪","e"],["⑪","f"],["⑪","g"],["⑪","h"],["⑪","◦"],["⑫","e"],["⑫","f"],["⑫","g"],["⑫","h"],["⑫","◦"],["⑬","e"],["⑬","f"],["⑬","g"],["⑬","h"],["⑬","◦"],["⑭","e"],["⑭","f"],["⑭","g"],["⑭","h"],["⑭","◦"],["█","·"],["█","█"]]}}

rule110-ether/tokenizer_config.json

@@ -0,0 +1 @@
+{"tokenizer_class": "PreTrainedTokenizerFast", "clean_up_tokenization_spaces": false, "model_max_length": 1000000000}

rule110/start.txt

@@ -0,0 +1 @@
+⊢◦·█⊣

rule110/tokenizer.json

@@ -0,0 +1 @@
+{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":42,"content":"··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":43,"content":"·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":44,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":45,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":46,"content":"··⊣","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":47,"content":"█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":48,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":49,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":50,"content":"█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":51,"content":"█·⊣","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":52,"content":"··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":53,"content":"·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":54,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":55,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":56,"content":"··⊣","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":57,"content":"█·","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":58,"content":"██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":59,"content":"·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":60,"content":"··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":61,"content":"█·⊣","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":62,"content":"ea","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":63,"content":"fb","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":64,"content":"⊢◦··","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":65,"content":"⊢◦·█","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":66,"content":"⊢◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":67,"content":"⊢◦██","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":68,"content":"⊢","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":69,"content":"gc","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":70,"content":"hd","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}],"normalizer":null,"pre_tokenizer":{"type":"Split","pattern":{"Regex":".."},"behavior":"Isolated","invert":false},"post_processor":null,"decoder":{"type":"Sequence","decoders":[{"type":"Replace","pattern":{"Regex":"[-]"},"content":""},{"type":"Fuse"}]},"model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":false,"vocab":{"a":0,"b":1,"c":2,"d":3,"e":4,"f":5,"g":6,"h":7,"·":8,"⊢":9,"⊣":10,"█":11,"◦":12,"ae":42,"af":43,"ag":44,"ah":45,"a⊣":46,"be":47,"bf":48,"bg":49,"bh":50,"b⊣":51,"ce":52,"cf":53,"cg":54,"ch":55,"c⊣":56,"de":57,"df":58,"dg":59,"dh":60,"d⊣":61,"··":62,"·█":63,"⊢e":64,"⊢f":65,"⊢g":66,"⊢h":67,"⊢◦":68,"█·":69,"██":70},"merges":[["a","e"],["a","f"],["a","g"],["a","h"],["a","⊣"],["b","e"],["b","f"],["b","g"],["b","h"],["b","⊣"],["c","e"],["c","f"],["c","g"],["c","h"],["c","⊣"],["d","e"],["d","f"],["d","g"],["d","h"],["d","⊣"],["·","·"],["·","█"],["⊢","e"],["⊢","f"],["⊢","g"],["⊢","h"],["⊢","◦"],["█","·"],["█","█"]]}}

rule110/tokenizer_config.json

@@ -0,0 +1 @@
+{"tokenizer_class": "PreTrainedTokenizerFast", "clean_up_tokenization_spaces": false, "model_max_length": 1000000000}