Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 shibatch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,462 @@
+---
+library_name: transformers
+pipeline_tag: text-generation
+language:
+  - en
+license: mit
+datasets:
+  - roneneldan/TinyStories
+tags:
+  - diffusion-gemma
+  - block-diffusion
+  - mixture-of-experts
+  - moe
+  - tinystories
+  - tiny-model
+  - text-generation
+  - validation
+  - debug-model
+---
+
+# Tiny Diffusion Gemma MoE 4M
+
+This repository contains a tiny, text-only `DiffusionGemmaForBlockDiffusion`
+Mixture-of-Experts language model trained from scratch on TinyStories.
+
+The model is intentionally small: it has **4,190,512 parameters** and an
+8-token diffusion canvas. It is primarily intended for validation, debugging,
+and experimentation with the Diffusion Gemma implementation in Hugging Face
+Transformers.
+
+This is a synthetic tiny checkpoint. It is not an official Google model, does
+not contain weights from an original Gemma checkpoint, and should not be
+expected to match the quality of production language models.
+
+## Repository contents
+
+- `hf/`: recommended EMA checkpoint using an exponential moving average of the
+  weights during the final 140,000-step training segment
+- `eval_text_generation.json`: final example generations
+- `artifact_metadata.json`: training arguments and evaluation metrics
+- `diffusion_gemma_config_dump.json`: expanded model configuration
+
+Only the recommended EMA checkpoint is included in this distribution package.
+Its aggregate validation metrics were slightly better than the final
+non-averaged checkpoint, and its sampled outputs were generally more readable
+in the final evaluation.
+
+## Model purpose
+
+This model is designed for:
+
+- testing `DiffusionGemmaConfig`
+- testing `DiffusionGemmaForBlockDiffusion`
+- checking block-diffusion generation
+- exercising first-pass and self-conditioned second-pass prediction
+- testing Diffusion Gemma MoE expert routing
+- exercising sliding-attention and full-attention layers
+- exercising grouped-query attention
+- validating custom tokenizer save/load behavior
+- checking `save_pretrained()` and `from_pretrained()`
+- providing a compact checkpoint for inference-engine development
+
+It is not designed for:
+
+- instruction following
+- chat use
+- high-quality long-form generation
+- benchmark comparison with production language models
+- production deployment
+- image input or multimodal use
+- reproducing the behavior of an official Gemma checkpoint
+
+## Model architecture
+
+The model uses `DiffusionGemmaForBlockDiffusion` with a small Diffusion
+Gemma-style encoder/decoder and MoE configuration.
+
+```yaml
+model_type: diffusion_gemma
+architecture: DiffusionGemmaForBlockDiffusion
+
+parameter_count: 4,190,512
+trainable_parameter_count: 4,147,360
+vocab_size: 1,003
+
+hidden_size: 128
+intermediate_size: 384
+moe_intermediate_size: 192
+
+num_hidden_layers: 8
+num_attention_heads: 4
+num_key_value_heads: 1
+num_global_key_value_heads: 1
+head_dim: 32
+global_head_dim: 32
+
+num_experts: 4
+top_k_experts: 2
+
+layer_pattern: sssssFsF
+sliding_window: 64
+max_position_embeddings: 1,024
+canvas_length: 8
+
+hidden_activation: gelu_pytorch_tanh
+tie_word_embeddings: true
+rms_norm_eps: 1.0e-6
+```
+
+The attention pattern is:
+
+```text
+sssssFsF
+```
+
+where `s` means `sliding_attention` and `F` means `full_attention`. The final
+layer uses full attention.
+
+The checkpoint includes the minimal vision modules required by the
+Diffusion Gemma configuration schema, but those modules were frozen and were
+not trained. This checkpoint is text-only.
+
+## MoE configuration
+
+Each decoder layer contains four local experts and routes each token to the
+top two experts:
+
+```yaml
+num_experts: 4
+top_k_experts: 2
+moe_intermediate_size: 192
+```
+
+This configuration keeps the checkpoint small while exercising router
+parameters, expert dispatch, weighted expert combination, and interaction
+between MoE blocks and both attention types.
+
+## Training data
+
+The model was trained on the full TinyStories training corpus:
+
+```yaml
+training_rows: 2,098,522
+validation_rows: 21,197
+validation_fraction: 0.01
+training_blocks: 9,764,172
+validation_blocks: 98,619
+block_size: 64
+```
+
+Stories were tokenized as a continuous stream:
+
+```text
+BOS + story 1 + EOS + BOS + story 2 + EOS + ...
+```
+
+The stream was packed into fixed 64-token blocks without padding. Only the
+final incomplete block was discarded.
+
+## Tokenizer
+
+The checkpoint uses a small custom byte-level BPE tokenizer. The base BPE
+vocabulary was trained with:
+
+```text
+BPE()
+ByteLevel(add_prefix_space=False)
+vocab_size: 1,000
+min_frequency: 2
+normalizer: None
+```
+
+Special tokens were added after BPE training:
+
+```text
+<s>          -> 1000
+</s>         -> 1001
+<|im_start|> -> 1002
+```
+
+The tokenizer uses:
+
+```yaml
+bos_token: <s>
+eos_token: </s>
+pad_token: <s>
+```
+
+Use `PreTrainedTokenizerFast` directly. `AutoTokenizer` may not infer this
+minimal custom tokenizer correctly in every Transformers version.
+
+## Training setup
+
+The checkpoint was trained in float32 on an NVIDIA GeForce GTX 1650.
+
+```yaml
+dtype: float32
+batch_size: 8
+gradient_accumulation_steps: 2
+effective_batch_size: 16
+block_size: 64
+
+ar_steps: 10,000
+partial_diffusion_steps: 5,000
+full_diffusion_steps_cumulative: 220,000
+
+ar_learning_rate: 2.0e-4
+diffusion_learning_rate_initial: 1.0e-4
+minimum_learning_rate: 1.0e-5
+
+adam_beta1: 0.9
+adam_beta2: 0.95
+adam_eps: 1.0e-8
+weight_decay: 0.01
+grad_clip: 1.0
+ema_decay: 0.999
+```
+
+Training used three phases:
+
+1. 10,000 autoregressive warmup updates.
+2. 5,000 partial-diffusion updates with at most four corrupted canvas tokens.
+3. 220,000 cumulative full-diffusion updates with up to all eight canvas
+   tokens corrupted.
+
+The original long run was interrupted after 89,483 full-diffusion updates.
+Training resumed from the complete 80,000-step checkpoint for another 140,000
+updates. The final checkpoint therefore represents 220,000 cumulative
+full-diffusion updates, but it is not an uninterrupted optimizer trajectory.
+The optimizer and EMA accumulator were restarted for the final 140,000-step
+segment.
+
+At batch size 8 with two gradient-accumulation steps, the cumulative
+full-diffusion phase consumed approximately 0.36 packed-data epochs.
+
+## Evaluation
+
+The final EMA checkpoint produced:
+
+```yaml
+autoregressive_validation_loss: 2.0143
+autoregressive_validation_perplexity: 7.4955
+```
+
+Diffusion evaluation used stochastic corruption and reports second-pass metrics
+over the complete 8-token canvas:
+
+| Corrupted canvas tokens | Validation loss | Accuracy |
+| ---: | ---: | ---: |
+| 2 of 8 | 0.6831 | 0.8613 |
+| 4 of 8 | 1.5972 | 0.6719 |
+| 6 of 8 | 3.1330 | 0.4316 |
+| 8 of 8 | 4.8636 | 0.1406 |
+
+The unweighted mean across these four corruption levels is approximately:
+
+```yaml
+mean_diffusion_validation_loss: 2.5692
+mean_diffusion_accuracy: 0.5264
+```
+
+These values were computed over eight validation batches and should be treated
+as compact checkpoint diagnostics, not as general language-model benchmarks.
+
+## Example generation
+
+Examples below come from the final EMA checkpoint with the saved generation
+settings.
+
+### Prompt: `Once upon`
+
+```text
+Once upon a time, there was a little girl. She loved to play on the swings and
+play. One day, she decided to go.
+```
+
+### Prompt: `There was a little`
+
+```text
+There was a little girl named Lily. She liked to play with her toys and spend
+all her toys in her room with her mommy. One day, she went to play with her
+toys and Lily saw a beautiful yellow berriy. She wanted to make it, but her
+mom said her it was right. Her
+```
+
+### Prompt: `One day`
+
+```text
+One day, a little girl named Mia and Ben went walking in the park. They saw a
+big block and a big car. Lily wanted to toub it, but she just wanted it.
+
+"Oh, Ben, what is that, doing?" Ben asked.
+
+"Well,
+```
+
+The model has learned recognizable TinyStories-style English, but malformed
+words, grammatical errors, unfinished sentences, repetition, and semantic
+drift are expected.
+
+## Usage
+
+This example loads the recommended EMA checkpoint from the `hf` subdirectory.
+
+```python
+import torch
+from transformers import (
+    DiffusionGemmaForBlockDiffusion,
+    DiffusionGemmaGenerationConfig,
+    EntropyBoundSamplerConfig,
+    PreTrainedTokenizerFast,
+)
+
+repo = "shibatch/tinydiffusiongemmamoe4m"
+subfolder = "hf"
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+tokenizer = PreTrainedTokenizerFast.from_pretrained(
+    repo,
+    subfolder=subfolder,
+)
+model = DiffusionGemmaForBlockDiffusion.from_pretrained(
+    repo,
+    subfolder=subfolder,
+    dtype=torch.float32,
+).to(device)
+model.eval()
+
+generation_config = DiffusionGemmaGenerationConfig(
+    max_new_tokens=64,
+    max_denoising_steps=64,
+    sampler_config=EntropyBoundSamplerConfig(entropy_bound=1.0),
+    t_min=0.4,
+    t_max=0.8,
+    stability_threshold=3,
+    confidence_threshold=0.05,
+    bos_token_id=tokenizer.bos_token_id,
+    eos_token_id=tokenizer.eos_token_id,
+    pad_token_id=tokenizer.pad_token_id,
+    cache_implementation="dynamic",
+    return_dict_in_generate=True,
+)
+
+prompt = "Once upon"
+input_ids = torch.tensor(
+    [[tokenizer.bos_token_id] + tokenizer.encode(
+        prompt,
+        add_special_tokens=False,
+    )],
+    dtype=torch.long,
+    device=device,
+)
+
+with torch.no_grad():
+    output = model.generate(
+        input_ids=input_ids,
+        generation_config=generation_config,
+    )
+
+sequences = output.sequences if hasattr(output, "sequences") else output
+print(tokenizer.decode(sequences[0].tolist(), skip_special_tokens=True))
+```
+
+## Loading requirements
+
+The checkpoint requires a Transformers release that includes:
+
+```python
+from transformers import (
+    DiffusionGemmaConfig,
+    DiffusionGemmaForBlockDiffusion,
+    DiffusionGemmaGenerationConfig,
+    EntropyBoundSamplerConfig,
+)
+```
+
+The checkpoint was trained and tested with:
+
+```text
+transformers 5.14.1
+torch 2.14.0.dev20260720+cu126
+```
+
+If these Diffusion Gemma imports fail, install a newer Transformers release
+that includes the architecture.
+
+The usage example constructs `DiffusionGemmaGenerationConfig` explicitly so
+that its nested `EntropyBoundSamplerConfig` is restored as the correct config
+class rather than as a plain dictionary.
+
+## Intended validation coverage
+
+This checkpoint is intended to validate support for:
+
+- `DiffusionGemmaConfig`
+- `DiffusionGemmaForBlockDiffusion`
+- `DiffusionGemmaGenerationConfig`
+- `EntropyBoundSamplerConfig`
+- block-diffusion generation with an 8-token canvas
+- first-pass prediction
+- detached self-conditioning logits
+- second-pass self-conditioned prediction
+- stochastic token corruption
+- sliding attention
+- full attention
+- GQA with one key/value head
+- four local MoE experts
+- top-2 expert routing
+- tied token embeddings
+- custom byte-level BPE loading
+- `generate()`
+- `save_pretrained()`
+- `from_pretrained()`
+
+## Limitations
+
+Known limitations include:
+
+- very small model capacity
+- small 1,003-token vocabulary
+- malformed or invented words
+- weak semantic consistency
+- weak long-form coherence
+- repetition and TinyStories template collapse
+- only an 8-token diffusion canvas
+- no instruction tuning
+- no chat template
+- no image or multimodal capability
+- no production-use claim
+- no quality-equivalence claim with official Gemma models
+- no current llama.cpp or GGUF inference support for this architecture
+
+The checkpoint is most useful for testing and improving Diffusion Gemma
+implementations without downloading a large model.
+
+## Why include MoE?
+
+A dense tiny model would be simpler, but it would not cover the MoE router,
+expert dispatch, top-k selection, or expert output-combination paths. Four
+experts with top-2 routing provide useful implementation coverage while keeping
+the checkpoint compact.
+
+## Why mix sliding and full attention?
+
+A full-attention-only model would not exercise Diffusion Gemma's local
+attention path. The `sssssFsF` pattern covers both implementations and keeps
+the final layer globally attentive.
+
+## Notes on GGUF and quantization
+
+The checkpoint is distributed as normal float32 Hugging Face Safetensors.
+Current llama.cpp conversion tools do not support
+`DiffusionGemmaForBlockDiffusion`, and llama.cpp does not provide the matching
+block-diffusion inference graph. A GGUF container could be produced with a
+custom converter, but it would not currently be usable for inference in
+llama.cpp.
+
+## Citation
+
+This is a synthetic tiny Diffusion Gemma-compatible MoE checkpoint trained
+from scratch on TinyStories. It is intended for implementation testing,
+debugging, and small-scale block-diffusion experiments.

artifact_metadata.json

@@ -0,0 +1,386 @@
+{
+  "model_type": "tiny_diffusion_gemma_moe_5m",
+  "hf_model_type": "diffusion_gemma",
+  "parameter_count": 4190512,
+  "trainable_parameter_count": 4147360,
+  "tokenizer_size": 1003,
+  "split_sizes": {
+    "ar_train_rows": 2098522,
+    "ar_validation_rows": 21197,
+    "diffusion_train_rows": 2098522,
+    "diffusion_validation_rows": 21197
+  },
+  "packed_sizes": {
+    "ar_train_blocks": 9764172,
+    "ar_validation_blocks": 98619,
+    "diffusion_train_blocks": 9764172,
+    "diffusion_validation_blocks": 98619
+  },
+  "phase_metrics": {
+    "phase_1_ar": {},
+    "phase_2_partial_diffusion": {},
+    "phase_3_full_diffusion": {
+      "noise_2_of_8_stochastic_first_accuracy": 0.42646167799830437,
+      "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875,
+      "noise_2_of_8_stochastic_first_loss": 2.6051205694675446,
+      "noise_2_of_8_stochastic_first_loss_full": 0.7259416617453098,
+      "noise_2_of_8_stochastic_second_accuracy": 0.4603654481470585,
+      "noise_2_of_8_stochastic_second_accuracy_full": 0.857421875,
+      "noise_2_of_8_stochastic_second_loss": 2.4278698414564133,
+      "noise_2_of_8_stochastic_second_loss_full": 0.68489970266819,
+      "noise_4_of_8_stochastic_first_accuracy": 0.3474049922078848,
+      "noise_4_of_8_stochastic_first_accuracy_full": 0.66015625,
+      "noise_4_of_8_stochastic_first_loss": 3.226193368434906,
+      "noise_4_of_8_stochastic_first_loss_full": 1.7175433486700058,
+      "noise_4_of_8_stochastic_second_accuracy": 0.3570522516965866,
+      "noise_4_of_8_stochastic_second_accuracy_full": 0.66796875,
+      "noise_4_of_8_stochastic_second_loss": 3.0200355648994446,
+      "noise_4_of_8_stochastic_second_loss_full": 1.6022306233644485,
+      "noise_6_of_8_stochastic_first_accuracy": 0.24033872783184052,
+      "noise_6_of_8_stochastic_first_accuracy_full": 0.4140625,
+      "noise_6_of_8_stochastic_first_loss": 4.164743721485138,
+      "noise_6_of_8_stochastic_first_loss_full": 3.2314010858535767,
+      "noise_6_of_8_stochastic_second_accuracy": 0.2592960111796856,
+      "noise_6_of_8_stochastic_second_accuracy_full": 0.4296875,
+      "noise_6_of_8_stochastic_second_loss": 4.050312131643295,
+      "noise_6_of_8_stochastic_second_loss_full": 3.140813171863556,
+      "noise_8_of_8_stochastic_first_accuracy": 0.130859375,
+      "noise_8_of_8_stochastic_first_accuracy_full": 0.130859375,
+      "noise_8_of_8_stochastic_first_loss": 4.9496771693229675,
+      "noise_8_of_8_stochastic_first_loss_full": 4.9496771693229675,
+      "noise_8_of_8_stochastic_second_accuracy": 0.142578125,
+      "noise_8_of_8_stochastic_second_accuracy_full": 0.142578125,
+      "noise_8_of_8_stochastic_second_loss": 4.875990211963654,
+      "noise_8_of_8_stochastic_second_loss_full": 4.875990211963654
+    }
+  },
+  "final_ar_validation_loss": 2.0182483792304993,
+  "final_ar_validation_ppl": 7.525132204842911,
+  "final_diffusion_metrics": {
+    "noise_2_of_8_stochastic_first_accuracy": 0.42646167799830437,
+    "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875,
+    "noise_2_of_8_stochastic_first_loss": 2.6051205694675446,
+    "noise_2_of_8_stochastic_first_loss_full": 0.7259416617453098,
+    "noise_2_of_8_stochastic_second_accuracy": 0.4603654481470585,
+    "noise_2_of_8_stochastic_second_accuracy_full": 0.857421875,
+    "noise_2_of_8_stochastic_second_loss": 2.4278698414564133,
+    "noise_2_of_8_stochastic_second_loss_full": 0.68489970266819,
+    "noise_4_of_8_stochastic_first_accuracy": 0.3474049922078848,
+    "noise_4_of_8_stochastic_first_accuracy_full": 0.66015625,
+    "noise_4_of_8_stochastic_first_loss": 3.226193368434906,
+    "noise_4_of_8_stochastic_first_loss_full": 1.7175433486700058,
+    "noise_4_of_8_stochastic_second_accuracy": 0.3570522516965866,
+    "noise_4_of_8_stochastic_second_accuracy_full": 0.66796875,
+    "noise_4_of_8_stochastic_second_loss": 3.0200355648994446,
+    "noise_4_of_8_stochastic_second_loss_full": 1.6022306233644485,
+    "noise_6_of_8_stochastic_first_accuracy": 0.24033872783184052,
+    "noise_6_of_8_stochastic_first_accuracy_full": 0.4140625,
+    "noise_6_of_8_stochastic_first_loss": 4.164743721485138,
+    "noise_6_of_8_stochastic_first_loss_full": 3.2314010858535767,
+    "noise_6_of_8_stochastic_second_accuracy": 0.2592960111796856,
+    "noise_6_of_8_stochastic_second_accuracy_full": 0.4296875,
+    "noise_6_of_8_stochastic_second_loss": 4.050312131643295,
+    "noise_6_of_8_stochastic_second_loss_full": 3.140813171863556,
+    "noise_8_of_8_stochastic_first_accuracy": 0.130859375,
+    "noise_8_of_8_stochastic_first_accuracy_full": 0.130859375,
+    "noise_8_of_8_stochastic_first_loss": 4.9496771693229675,
+    "noise_8_of_8_stochastic_first_loss_full": 4.9496771693229675,
+    "noise_8_of_8_stochastic_second_accuracy": 0.142578125,
+    "noise_8_of_8_stochastic_second_accuracy_full": 0.142578125,
+    "noise_8_of_8_stochastic_second_loss": 4.875990211963654,
+    "noise_8_of_8_stochastic_second_loss_full": 4.875990211963654
+  },
+  "final_ema_ar_validation_loss": 2.014301896095276,
+  "final_ema_ar_validation_ppl": 7.4954929214408565,
+  "final_ema_diffusion_metrics": {
+    "noise_2_of_8_stochastic_first_accuracy": 0.42535873129963875,
+    "noise_2_of_8_stochastic_first_accuracy_full": 0.85546875,
+    "noise_2_of_8_stochastic_first_loss": 2.603432834148407,
+    "noise_2_of_8_stochastic_first_loss_full": 0.7258147932589054,
+    "noise_2_of_8_stochastic_second_accuracy": 0.47312964498996735,
+    "noise_2_of_8_stochastic_second_accuracy_full": 0.861328125,
+    "noise_2_of_8_stochastic_second_loss": 2.423415556550026,
+    "noise_2_of_8_stochastic_second_loss_full": 0.6831153854727745,
+    "noise_4_of_8_stochastic_first_accuracy": 0.3335161041468382,
+    "noise_4_of_8_stochastic_first_accuracy_full": 0.658203125,
+    "noise_4_of_8_stochastic_first_loss": 3.2337255477905273,
+    "noise_4_of_8_stochastic_first_loss_full": 1.7171394303441048,
+    "noise_4_of_8_stochastic_second_accuracy": 0.3573715351521969,
+    "noise_4_of_8_stochastic_second_accuracy_full": 0.671875,
+    "noise_4_of_8_stochastic_second_loss": 3.0173322558403015,
+    "noise_4_of_8_stochastic_second_loss_full": 1.5972065776586533,
+    "noise_6_of_8_stochastic_first_accuracy": 0.24319838173687458,
+    "noise_6_of_8_stochastic_first_accuracy_full": 0.416015625,
+    "noise_6_of_8_stochastic_first_loss": 4.165034174919128,
+    "noise_6_of_8_stochastic_first_loss_full": 3.231293201446533,
+    "noise_6_of_8_stochastic_second_accuracy": 0.2591778077185154,
+    "noise_6_of_8_stochastic_second_accuracy_full": 0.431640625,
+    "noise_6_of_8_stochastic_second_loss": 4.044229626655579,
+    "noise_6_of_8_stochastic_second_loss_full": 3.13301882147789,
+    "noise_8_of_8_stochastic_first_accuracy": 0.12890625,
+    "noise_8_of_8_stochastic_first_accuracy_full": 0.12890625,
+    "noise_8_of_8_stochastic_first_loss": 4.936682164669037,
+    "noise_8_of_8_stochastic_first_loss_full": 4.936682164669037,
+    "noise_8_of_8_stochastic_second_accuracy": 0.140625,
+    "noise_8_of_8_stochastic_second_accuracy_full": 0.140625,
+    "noise_8_of_8_stochastic_second_loss": 4.863627254962921,
+    "noise_8_of_8_stochastic_second_loss_full": 4.863627254962921
+  },
+  "normal_generations": {
+    "Once upon": "Once upon a time, there was a happy voice. He was very kindful and calm. He had made his friends who wrap them the time. One day, he saw a new geater in his bright flower. He thought a special game, so it could dry",
+    "There was a little": "There was a little girl named Lily. She liked to play with her toys and spend all her things inls with her friends. One day, Lily's.",
+    "One day": "One day, a big harage named Tim wanted to play near it. He saw a big mottle in the lard. He wanted to say sorry to his friend, \"Rrue, Tim. He ruffed you. He was very curious. He looked around his play"
+  },
+  "ema_generations": {
+    "Once upon": "Once upon a time, there was a little girl. She loved to play on the swings and play. One day, she decided to go.",
+    "There was a little": "There was a little girl named Lily. She liked to play with her toys and spend all her toys in her room with her mommy. One day, she went to play with her toys and Lily saw a beautiful yellow berriy. She wanted to make it, but her mom said her it was right. Her",
+    "One day": "One day, a little girl named Mia and Ben went walking in the park. They saw a big block and a big car. Lily wanted to toub it, but she just wanted it.\n\n\"Oh, Ben, what is that, doing?\" Ben asked.\n\n\"Well,"
+  },
+  "args": {
+    "output_dir": "runs/diffusion_gemma_100p_full220k_resume80k",
+    "init_model_dir": "runs/diffusion_gemma_100p_scratch_full220k/checkpoints/full-0080000/hf",
+    "train_files": [
+      "data/train-00000-of-00004-2d5a1467fff1081b.parquet",
+      "data/train-00001-of-00004-5852b56a2bd28fd9.parquet",
+      "data/train-00002-of-00004-a26307300439e943.parquet",
+      "data/train-00003-of-00004-d243063613e5a057.parquet"
+    ],
+    "max_rows": 0,
+    "data_fraction": 1.0,
+    "validation_fraction": 0.01,
+    "disjoint_phase_data": false,
+    "vocab_size": 1024,
+    "base_vocab_size": 1000,
+    "min_frequency": 2,
+    "legacy_tokenizer": true,
+    "legacy_special_token_ids": true,
+    "hidden_size": 128,
+    "intermediate_size": 384,
+    "moe_intermediate_size": 192,
+    "num_hidden_layers": 8,
+    "num_attention_heads": 4,
+    "num_key_value_heads": 1,
+    "num_global_key_value_heads": 1,
+    "head_dim": 32,
+    "global_head_dim": 32,
+    "num_experts": 4,
+    "top_k_experts": 2,
+    "sliding_window": 64,
+    "max_position_embeddings": 1024,
+    "layer_pattern": "sssssFsF",
+    "canvas_length": 8,
+    "max_parameters": 5000000,
+    "vision_hidden_size": 32,
+    "vision_intermediate_size": 64,
+    "vision_num_hidden_layers": 1,
+    "vision_num_attention_heads": 4,
+    "vision_head_dim": 8,
+    "vision_position_embedding_size": 64,
+    "block_size": 64,
+    "batch_size": 8,
+    "gradient_accumulation_steps": 2,
+    "ar_steps": 0,
+    "partial_diffusion_steps": 0,
+    "full_diffusion_steps": 140000,
+    "ar_learning_rate": 0.0002,
+    "diffusion_learning_rate": 7.41e-05,
+    "minimum_learning_rate": 1e-05,
+    "ar_warmup_steps": 1000,
+    "partial_warmup_steps": 500,
+    "full_warmup_steps": 0,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-08,
+    "weight_decay": 0.01,
+    "grad_clip": 1.0,
+    "error_on_nonfinite_gradients": true,
+    "first_pass_loss_weight": 1.0,
+    "clean_position_loss_weight": 1.0,
+    "partial_ar_aux_weight": 1.0,
+    "full_ar_aux_weight": 1.0,
+    "partial_self_conditioning_probability": 1.0,
+    "full_self_conditioning_probability": 1.0,
+    "separate_noise_for_second_pass": false,
+    "ema_decay": 0.999,
+    "min_corruption_rate": 0.0,
+    "max_corruption_rate": 1.0,
+    "eval_stochastic_corruption": true,
+    "log_steps": 50,
+    "eval_steps": 1000,
+    "eval_batches": 8,
+    "save_steps": 10000,
+    "seed": 1234,
+    "device": "cuda",
+    "dtype": "float32",
+    "num_workers": 0,
+    "generate_prompts": [
+      "Once upon",
+      "There was a little",
+      "One day"
+    ],
+    "generate_new_tokens": 64,
+    "max_denoising_steps": 64,
+    "entropy_bound": 1.0,
+    "t_min": 0.4,
+    "t_max": 0.8,
+    "stability_threshold": 3,
+    "confidence_threshold": 0.05
+  },
+  "config": {
+    "transformers_version": "5.14.1",
+    "architectures": [
+      "DiffusionGemmaForBlockDiffusion"
+    ],
+    "output_hidden_states": false,
+    "return_dict": true,
+    "dtype": "float32",
+    "chunk_size_feed_forward": 0,
+    "is_encoder_decoder": false,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "problem_type": null,
+    "text_config": {
+      "architectures": null,
+      "output_hidden_states": false,
+      "return_dict": true,
+      "dtype": "float32",
+      "chunk_size_feed_forward": 0,
+      "is_encoder_decoder": false,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "problem_type": null,
+      "vocab_size": 1003,
+      "hidden_size": 128,
+      "intermediate_size": 384,
+      "num_hidden_layers": 8,
+      "num_attention_heads": 4,
+      "num_key_value_heads": 1,
+      "head_dim": 32,
+      "hidden_activation": "gelu_pytorch_tanh",
+      "max_position_embeddings": 1024,
+      "initializer_range": 0.02,
+      "rms_norm_eps": 1e-06,
+      "pad_token_id": 1000,
+      "eos_token_id": 1001,
+      "bos_token_id": 1000,
+      "tie_word_embeddings": true,
+      "rope_parameters": {
+        "full_attention": {
+          "partial_rotary_factor": 0.25,
+          "rope_theta": 1000000.0,
+          "rope_type": "proportional"
+        },
+        "sliding_attention": {
+          "rope_theta": 10000.0,
+          "rope_type": "default"
+        }
+      },
+      "attention_bias": false,
+      "attention_dropout": 0.0,
+      "sliding_window": 64,
+      "layer_types": [
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "sliding_attention",
+        "full_attention",
+        "sliding_attention",
+        "full_attention"
+      ],
+      "use_bidirectional_attention": "vision",
+      "num_global_key_value_heads": 1,
+      "global_head_dim": 32,
+      "num_experts": 4,
+      "top_k_experts": 2,
+      "moe_intermediate_size": 192,
+      "_name_or_path": "",
+      "model_type": "diffusion_gemma_text",
+      "output_attentions": false
+    },
+    "vision_config": {
+      "architectures": null,
+      "output_hidden_states": false,
+      "return_dict": true,
+      "dtype": "float32",
+      "chunk_size_feed_forward": 0,
+      "is_encoder_decoder": false,
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "problem_type": null,
+      "hidden_size": 32,
+      "intermediate_size": 64,
+      "num_hidden_layers": 1,
+      "num_attention_heads": 4,
+      "num_key_value_heads": 4,
+      "head_dim": 8,
+      "hidden_activation": "gelu_pytorch_tanh",
+      "rms_norm_eps": 1e-06,
+      "max_position_embeddings": 256,
+      "attention_bias": false,
+      "attention_dropout": 0.0,
+      "rope_parameters": {
+        "rope_theta": 100.0,
+        "rope_type": "default"
+      },
+      "pooling_kernel_size": 1,
+      "patch_size": 16,
+      "position_embedding_size": 64,
+      "use_clipped_linears": false,
+      "standardize": false,
+      "initializer_range": 0.02,
+      "_name_or_path": "",
+      "model_type": "gemma4_vision",
+      "output_attentions": false
+    },
+    "boi_token_id": 255999,
+    "eoi_token_id": 258882,
+    "image_token_id": 258880,
+    "initializer_range": 0.02,
+    "tie_word_embeddings": true,
+    "canvas_length": 8,
+    "_name_or_path": "/home/codex/tinyModels/runs/diffusion_gemma_100p_scratch_full220k/checkpoints/full-0080000/hf",
+    "model_type": "diffusion_gemma",
+    "output_attentions": false
+  },
+  "generation_config": {
+    "max_new_tokens": 64,
+    "max_length": null,
+    "return_dict_in_generate": true,
+    "max_denoising_steps": 64,
+    "sampler_config": {
+      "entropy_bound": 1.0,
+      "_cls_name": "EntropyBoundSamplerConfig"
+    },
+    "t_min": 0.4,
+    "t_max": 0.8,
+    "stability_threshold": 3,
+    "confidence_threshold": 0.05,
+    "cache_implementation": "dynamic",
+    "cache_config": null,
+    "disable_compile": null,
+    "bos_token_id": 1000,
+    "pad_token_id": 1000,
+    "eos_token_id": 1001,
+    "_from_model_config": null,
+    "transformers_version": "5.14.1"
+  }
+}

diffusion_gemma_config_dump.json

@@ -0,0 +1,135 @@
+{
+  "transformers_version": "5.14.1",
+  "architectures": [
+    "DiffusionGemmaForBlockDiffusion"
+  ],
+  "output_hidden_states": false,
+  "return_dict": true,
+  "dtype": "float32",
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "problem_type": null,
+  "text_config": {
+    "architectures": null,
+    "output_hidden_states": false,
+    "return_dict": true,
+    "dtype": "float32",
+    "chunk_size_feed_forward": 0,
+    "is_encoder_decoder": false,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "problem_type": null,
+    "vocab_size": 1003,
+    "hidden_size": 128,
+    "intermediate_size": 384,
+    "num_hidden_layers": 8,
+    "num_attention_heads": 4,
+    "num_key_value_heads": 1,
+    "head_dim": 32,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "max_position_embeddings": 1024,
+    "initializer_range": 0.02,
+    "rms_norm_eps": 1e-06,
+    "pad_token_id": 1000,
+    "eos_token_id": 1001,
+    "bos_token_id": 1000,
+    "tie_word_embeddings": true,
+    "rope_parameters": {
+      "full_attention": {
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 1000000.0,
+        "rope_type": "proportional"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "sliding_window": 64,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
+    "use_bidirectional_attention": "vision",
+    "num_global_key_value_heads": 1,
+    "global_head_dim": 32,
+    "num_experts": 4,
+    "top_k_experts": 2,
+    "moe_intermediate_size": 192,
+    "_name_or_path": "",
+    "model_type": "diffusion_gemma_text",
+    "output_attentions": false
+  },
+  "vision_config": {
+    "architectures": null,
+    "output_hidden_states": false,
+    "return_dict": true,
+    "dtype": "float32",
+    "chunk_size_feed_forward": 0,
+    "is_encoder_decoder": false,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "problem_type": null,
+    "hidden_size": 32,
+    "intermediate_size": 64,
+    "num_hidden_layers": 1,
+    "num_attention_heads": 4,
+    "num_key_value_heads": 4,
+    "head_dim": 8,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "rms_norm_eps": 1e-06,
+    "max_position_embeddings": 256,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "rope_parameters": {
+      "rope_theta": 100.0,
+      "rope_type": "default"
+    },
+    "pooling_kernel_size": 1,
+    "patch_size": 16,
+    "position_embedding_size": 64,
+    "use_clipped_linears": false,
+    "standardize": false,
+    "initializer_range": 0.02,
+    "_name_or_path": "",
+    "model_type": "gemma4_vision",
+    "output_attentions": false
+  },
+  "boi_token_id": 255999,
+  "eoi_token_id": 258882,
+  "image_token_id": 258880,
+  "initializer_range": 0.02,
+  "tie_word_embeddings": true,
+  "canvas_length": 8,
+  "_name_or_path": "/home/codex/tinyModels/runs/diffusion_gemma_100p_scratch_full220k/checkpoints/full-0080000/hf",
+  "model_type": "diffusion_gemma",
+  "output_attentions": false
+}

eval_text_generation.json

@@ -0,0 +1,12 @@
+{
+  "normal": {
+    "Once upon": "Once upon a time, there was a happy voice. He was very kindful and calm. He had made his friends who wrap them the time. One day, he saw a new geater in his bright flower. He thought a special game, so it could dry",
+    "There was a little": "There was a little girl named Lily. She liked to play with her toys and spend all her things inls with her friends. One day, Lily's.",
+    "One day": "One day, a big harage named Tim wanted to play near it. He saw a big mottle in the lard. He wanted to say sorry to his friend, \"Rrue, Tim. He ruffed you. He was very curious. He looked around his play"
+  },
+  "ema": {
+    "Once upon": "Once upon a time, there was a little girl. She loved to play on the swings and play. One day, she decided to go.",
+    "There was a little": "There was a little girl named Lily. She liked to play with her toys and spend all her toys in her room with her mommy. One day, she went to play with her toys and Lily saw a beautiful yellow berriy. She wanted to make it, but her mom said her it was right. Her",
+    "One day": "One day, a little girl named Mia and Ben went walking in the park. They saw a big block and a big car. Lily wanted to toub it, but she just wanted it.\n\n\"Oh, Ben, what is that, doing?\" Ben asked.\n\n\"Well,"
+  }
+}

example_generate.py

@@ -0,0 +1,62 @@
+import torch
+from transformers import (
+    DiffusionGemmaForBlockDiffusion,
+    DiffusionGemmaGenerationConfig,
+    EntropyBoundSamplerConfig,
+    PreTrainedTokenizerFast,
+)
+
+
+MODEL_PATH = "."
+MODEL_SUBFOLDER = "hf"
+PROMPT = "Once upon"
+
+
+def main() -> None:
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    tokenizer = PreTrainedTokenizerFast.from_pretrained(
+        MODEL_PATH,
+        subfolder=MODEL_SUBFOLDER,
+    )
+    model = DiffusionGemmaForBlockDiffusion.from_pretrained(
+        MODEL_PATH,
+        subfolder=MODEL_SUBFOLDER,
+        dtype=torch.float32,
+    ).to(device)
+    model.eval()
+
+    generation_config = DiffusionGemmaGenerationConfig(
+        max_new_tokens=64,
+        max_denoising_steps=64,
+        sampler_config=EntropyBoundSamplerConfig(entropy_bound=1.0),
+        t_min=0.4,
+        t_max=0.8,
+        stability_threshold=3,
+        confidence_threshold=0.05,
+        bos_token_id=tokenizer.bos_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+        pad_token_id=tokenizer.pad_token_id,
+        cache_implementation="dynamic",
+        return_dict_in_generate=True,
+    )
+    input_ids = torch.tensor(
+        [
+            [tokenizer.bos_token_id]
+            + tokenizer.encode(PROMPT, add_special_tokens=False)
+        ],
+        dtype=torch.long,
+        device=device,
+    )
+
+    with torch.no_grad():
+        output = model.generate(
+            input_ids=input_ids,
+            generation_config=generation_config,
+        )
+
+    sequences = output.sequences if hasattr(output, "sequences") else output
+    print(tokenizer.decode(sequences[0].tolist(), skip_special_tokens=True))
+
+
+if __name__ == "__main__":
+    main()

hf/config.json

@@ -0,0 +1,104 @@
+{
+  "architectures": [
+    "DiffusionGemmaForBlockDiffusion"
+  ],
+  "boi_token_id": 255999,
+  "canvas_length": 8,
+  "dtype": "float32",
+  "eoi_token_id": 258882,
+  "image_token_id": 258880,
+  "initializer_range": 0.02,
+  "model_type": "diffusion_gemma",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 1000,
+    "dtype": "float32",
+    "eos_token_id": 1001,
+    "global_head_dim": 32,
+    "head_dim": 32,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 128,
+    "initializer_range": 0.02,
+    "intermediate_size": 384,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 1024,
+    "model_type": "diffusion_gemma_text",
+    "moe_intermediate_size": 192,
+    "num_attention_heads": 4,
+    "num_experts": 4,
+    "num_global_key_value_heads": 1,
+    "num_hidden_layers": 8,
+    "num_key_value_heads": 1,
+    "pad_token_id": 1000,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "full_attention": {
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 1000000.0,
+        "rope_type": "proportional"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 64,
+    "tie_word_embeddings": true,
+    "top_k_experts": 2,
+    "use_bidirectional_attention": "vision",
+    "vocab_size": 1003
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "vision_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "chunk_size_feed_forward": 0,
+    "dtype": "float32",
+    "head_dim": 8,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 32,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 64,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "max_position_embeddings": 256,
+    "model_type": "gemma4_vision",
+    "num_attention_heads": 4,
+    "num_hidden_layers": 1,
+    "num_key_value_heads": 4,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "patch_size": 16,
+    "pooling_kernel_size": 1,
+    "position_embedding_size": 64,
+    "problem_type": null,
+    "return_dict": true,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 100.0,
+      "rope_type": "default"
+    },
+    "standardize": false,
+    "use_clipped_linears": false
+  }
+}

hf/generation_config.json

@@ -0,0 +1,18 @@
+{
+  "bos_token_id": 1000,
+  "cache_implementation": "dynamic",
+  "confidence_threshold": 0.05,
+  "eos_token_id": 1001,
+  "max_denoising_steps": 64,
+  "max_new_tokens": 64,
+  "pad_token_id": 1000,
+  "return_dict_in_generate": true,
+  "sampler_config": {
+    "_cls_name": "EntropyBoundSamplerConfig",
+    "entropy_bound": 1.0
+  },
+  "stability_threshold": 3,
+  "t_max": 0.8,
+  "t_min": 0.4,
+  "transformers_version": "5.14.1"
+}

hf/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47aad32112dba5e1f4b68fd2cfb90ef9b5958b184aa7970da4107072713643ea
+size 16785984

hf/tokenizer.json

@@ -0,0 +1,4063 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 1000,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1001,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1002,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {}
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "!": 0,
+      "\"": 1,
+      "#": 2,
+      "$": 3,
+      "%": 4,
+      "&": 5,
+      "'": 6,
+      "(": 7,
+      ")": 8,
+      "*": 9,
+      "+": 10,
+      ",": 11,
+      "-": 12,
+      ".": 13,
+      "/": 14,
+      "0": 15,
+      "1": 16,
+      "2": 17,
+      "3": 18,
+      "4": 19,
+      "5": 20,
+      "6": 21,
+      "7": 22,
+      "8": 23,
+      "9": 24,
+      ":": 25,
+      ";": 26,
+      "<": 27,
+      "=": 28,
+      ">": 29,
+      "?": 30,
+      "@": 31,
+      "A": 32,
+      "B": 33,
+      "C": 34,
+      "D": 35,
+      "E": 36,
+      "F": 37,
+      "G": 38,
+      "H": 39,
+      "I": 40,
+      "J": 41,
+      "K": 42,
+      "L": 43,
+      "M": 44,
+      "N": 45,
+      "O": 46,
+      "P": 47,
+      "Q": 48,
+      "R": 49,
+      "S": 50,
+      "T": 51,
+      "U": 52,
+      "V": 53,
+      "W": 54,
+      "X": 55,
+      "Y": 56,
+      "Z": 57,
+      "[": 58,
+      "\\": 59,
+      "]": 60,
+      "^": 61,
+      "_": 62,
+      "`": 63,
+      "a": 64,
+      "b": 65,
+      "c": 66,
+      "d": 67,
+      "e": 68,
+      "f": 69,
+      "g": 70,
+      "h": 71,
+      "i": 72,
+      "j": 73,
+      "k": 74,
+      "l": 75,
+      "m": 76,
+      "n": 77,
+      "o": 78,
+      "p": 79,
+      "q": 80,
+      "r": 81,
+      "s": 82,
+      "t": 83,
+      "u": 84,
+      "v": 85,
+      "w": 86,
+      "x": 87,
+      "y": 88,
+      "z": 89,
+      "{": 90,
+      "|": 91,
+      "}": 92,
+      "~": 93,
+      "¡": 94,
+      "¢": 95,
+      "£": 96,
+      "¤": 97,
+      "¥": 98,
+      "¦": 99,
+      "§": 100,
+      "¨": 101,
+      "©": 102,
+      "ª": 103,
+      "«": 104,
+      "¬": 105,
+      "®": 106,
+      "¯": 107,
+      "°": 108,
+      "±": 109,
+      "²": 110,
+      "³": 111,
+      "´": 112,
+      "µ": 113,
+      "¶": 114,
+      "·": 115,
+      "¸": 116,
+      "¹": 117,
+      "º": 118,
+      "»": 119,
+      "¼": 120,
+      "½": 121,
+      "¾": 122,
+      "¿": 123,
+      "À": 124,
+      "Á": 125,
+      "Â": 126,
+      "Ã": 127,
+      "Ä": 128,
+      "Å": 129,
+      "Æ": 130,
+      "Ç": 131,
+      "È": 132,
+      "É": 133,
+      "Ê": 134,
+      "Ë": 135,
+      "Ì": 136,
+      "Í": 137,
+      "Î": 138,
+      "Ï": 139,
+      "Ð": 140,
+      "Ñ": 141,
+      "Ò": 142,
+      "Ó": 143,
+      "Ô": 144,
+      "Õ": 145,
+      "Ö": 146,
+      "×": 147,
+      "Ø": 148,
+      "Ù": 149,
+      "Ú": 150,
+      "Û": 151,
+      "Ü": 152,
+      "Ý": 153,
+      "Þ": 154,
+      "ß": 155,
+      "à": 156,
+      "á": 157,
+      "â": 158,
+      "ã": 159,
+      "ä": 160,
+      "å": 161,
+      "æ": 162,
+      "ç": 163,
+      "è": 164,
+      "é": 165,
+      "ê": 166,
+      "ë": 167,
+      "ì": 168,
+      "í": 169,
+      "î": 170,
+      "ï": 171,
+      "ð": 172,
+      "ñ": 173,
+      "ò": 174,
+      "ó": 175,
+      "ô": 176,
+      "õ": 177,
+      "ö": 178,
+      "÷": 179,
+      "ø": 180,
+      "ù": 181,
+      "ú": 182,
+      "û": 183,
+      "ü": 184,
+      "ý": 185,
+      "þ": 186,
+      "ÿ": 187,
+      "Ā": 188,
+      "ā": 189,
+      "Ă": 190,
+      "ă": 191,
+      "Ą": 192,
+      "ą": 193,
+      "Ć": 194,
+      "ć": 195,
+      "Ĉ": 196,
+      "ĉ": 197,
+      "Ċ": 198,
+      "ċ": 199,
+      "Č": 200,
+      "č": 201,
+      "Ď": 202,
+      "ď": 203,
+      "Đ": 204,
+      "đ": 205,
+      "Ē": 206,
+      "ē": 207,
+      "Ĕ": 208,
+      "ĕ": 209,
+      "Ė": 210,
+      "ė": 211,
+      "Ę": 212,
+      "ę": 213,
+      "Ě": 214,
+      "ě": 215,
+      "Ĝ": 216,
+      "ĝ": 217,
+      "Ğ": 218,
+      "ğ": 219,
+      "Ġ": 220,
+      "ġ": 221,
+      "Ģ": 222,
+      "ģ": 223,
+      "Ĥ": 224,
+      "ĥ": 225,
+      "Ħ": 226,
+      "ħ": 227,
+      "Ĩ": 228,
+      "ĩ": 229,
+      "Ī": 230,
+      "ī": 231,
+      "Ĭ": 232,
+      "ĭ": 233,
+      "Į": 234,
+      "į": 235,
+      "İ": 236,
+      "ı": 237,
+      "IJ": 238,
+      "ij": 239,
+      "Ĵ": 240,
+      "ĵ": 241,
+      "Ķ": 242,
+      "ķ": 243,
+      "ĸ": 244,
+      "Ĺ": 245,
+      "ĺ": 246,
+      "Ļ": 247,
+      "ļ": 248,
+      "Ľ": 249,
+      "ľ": 250,
+      "Ŀ": 251,
+      "ŀ": 252,
+      "Ł": 253,
+      "ł": 254,
+      "Ń": 255,
+      "he": 256,
+      "Ġt": 257,
+      "Ġa": 258,
+      "Ġs": 259,
+      "Ġw": 260,
+      "nd": 261,
+      "Ġthe": 262,
+      "ed": 263,
+      "Ġand": 264,
+      "Ġto": 265,
+      "Ġb": 266,
+      "in": 267,
+      "Ġh": 268,
+      "Ġwa": 269,
+      "re": 270,
+      "ou": 271,
+      "Ġf": 272,
+      "it": 273,
+      "Ġl": 274,
+      "Ġc": 275,
+      "Ġd": 276,
+      "er": 277,
+      "Ġhe": 278,
+      "Ġm": 279,
+      "Ġp": 280,
+      "Ġwas": 281,
+      "om": 282,
+      "ĠT": 283,
+      "Ġo": 284,
+      "ay": 285,
+      "ar": 286,
+      "is": 287,
+      "ing": 288,
+      "il": 289,
+      "Ġg": 290,
+      "id": 291,
+      "at": 292,
+      "en": 293,
+      "Ġn": 294,
+      "Ġsa": 295,
+      "Ġha": 296,
+      "ĠS": 297,
+      "ĠThe": 298,
+      "an": 299,
+      "im": 300,
+      "or": 301,
+      "on": 302,
+      "Ġit": 303,
+      "Ġth": 304,
+      "ll": 305,
+      "le": 306,
+      "ĠH": 307,
+      "Ġher": 308,
+      "ot": 309,
+      "et": 310,
+      "ir": 311,
+      "ĠShe": 312,
+      "es": 313,
+      "ĠHe": 314,
+      "Ġin": 315,
+      "ver": 316,
+      "ut": 317,
+      "ow": 318,
+      "ĠThey": 319,
+      "ck": 320,
+      "Ġe": 321,
+      "Ġu": 322,
+      "ld": 323,
+      "Ġy": 324,
+      "oo": 325,
+      "Ġsaid": 326,
+      "ig": 327,
+      "Ġ\"": 328,
+      "ily": 329,
+      "Ġr": 330,
+      "Ġbe": 331,
+      "am": 332,
+      "Ġst": 333,
+      "ĠI": 334,
+      "ce": 335,
+      "ke": 336,
+      "Ġshe": 337,
+      "ve": 338,
+      "pp": 339,
+      "ith": 340,
+      "Ġwith": 341,
+      "Lily": 342,
+      "On": 343,
+      "Ġon": 344,
+      "Ġyou": 345,
+      "Ġof": 346,
+      "ked": 347,
+      "ri": 348,
+      "Ġso": 349,
+      "nt": 350,
+      "Ġhis": 351,
+      "Ġpl": 352,
+      "ad": 353,
+      "very": 354,
+      "Ġday": 355,
+      "Ġthat": 356,
+      "Ġup": 357,
+      "Ġplay": 358,
+      "Ġhad": 359,
+      "st": 360,
+      "Ġthey": 361,
+      "Ġwe": 362,
+      "ĠLily": 363,
+      "Ġmom": 364,
+      "Ġfor": 365,
+      "ĠB": 366,
+      "el": 367,
+      "un": 368,
+      "ould": 369,
+      "'s": 370,
+      "my": 371,
+      "The": 372,
+      "Ġli": 373,
+      "ch": 374,
+      "ent": 375,
+      "Ġhapp": 376,
+      "Ġwant": 377,
+      "itt": 378,
+      "Ġsh": 379,
+      "out": 380,
+      "her": 381,
+      "Ġdo": 382,
+      "ly": 383,
+      "Ġnot": 384,
+      "ome": 385,
+      "se": 386,
+      "Ġvery": 387,
+      "ound": 388,
+      "end": 389,
+      "all": 390,
+      "Ġwh": 391,
+      "ime": 392,
+      "ittle": 393,
+      "Ġk": 394,
+      "Ġtime": 395,
+      "Ġlittle": 396,
+      "al": 397,
+      "ĠĊ": 398,
+      "Ġthere": 399,
+      "Ġne": 400,
+      "ht": 401,
+      "ĠM": 402,
+      "Ġre": 403,
+      "Ġhappy": 404,
+      "Ġsm": 405,
+      "Ġbig": 406,
+      "Ġis": 407,
+      "ĠIt": 408,
+      "Ġbut": 409,
+      "ack": 410,
+      "Ġan": 411,
+      "Ġsaw": 412,
+      "riend": 413,
+      ".\"": 414,
+      "Ġfriend": 415,
+      "ry": 416,
+      "Ġas": 417,
+      "ra": 418,
+      "One": 419,
+      "ide": 420,
+      "ake": 421,
+      "'t": 422,
+      "!\"": 423,
+      "ug": 424,
+      "Ġloo": 425,
+      "ved": 426,
+      "ter": 427,
+      "Ġwere": 428,
+      "ore": 429,
+      "Once": 430,
+      "Ġlo": 431,
+      "ec": 432,
+      "Ġhim": 433,
+      "Ġtoo": 434,
+      "Ġgo": 435,
+      "Ġbo": 436,
+      "Ġse": 437,
+      "irl": 438,
+      "Ġare": 439,
+      "Ġj": 440,
+      "ĠTim": 441,
+      "ard": 442,
+      "Ġwanted": 443,
+      "ill": 444,
+      "Ġgirl": 445,
+      "Ġupon": 446,
+      "Ġthem": 447,
+      "Ġout": 448,
+      "Ġat": 449,
+      "ur": 450,
+      "ind": 451,
+      "Ġtheir": 452,
+      "fu": 453,
+      "Ġsp": 454,
+      "way": 455,
+      "ĠA": 456,
+      "Ġsmil": 457,
+      "Ġdid": 458,
+      "Ġcould": 459,
+      "hen": 460,
+      "Ġhave": 461,
+      "ain": 462,
+      "art": 463,
+      "Ġex": 464,
+      "hed": 465,
+      "rom": 466,
+      "Ġwent": 467,
+      "Ġar": 468,
+      "ic": 469,
+      "Ġcan": 470,
+      "ĠJ": 471,
+      "lp": 472,
+      "ful": 473,
+      "ĠBen": 474,
+      "ood": 475,
+      "?\"": 476,
+      "hing": 477,
+      "Ġfriends": 478,
+      "Ġnam": 479,
+      "ight": 480,
+      "Ġall": 481,
+      "Ġhelp": 482,
+      "Ġkn": 483,
+      "um": 484,
+      "one": 485,
+      "ark": 486,
+      "Ġback": 487,
+      "Ġsay": 488,
+      "You": 489,
+      "Ġfun": 490,
+      "Ġcl": 491,
+      "ĠTom": 492,
+      "are": 493,
+      "Ġno": 494,
+      "Ġle": 495,
+      "op": 496,
+      "Ġsmiled": 497,
+      "Ġsc": 498,
+      "Ġnamed": 499,
+      "Ġal": 500,
+      "oug": 501,
+      "Ġloved": 502,
+      "side": 503,
+      "elt": 504,
+      "ĠOne": 505,
+      "Ġman": 506,
+      "Ġasked": 507,
+      "ick": 508,
+      "Ġtoy": 509,
+      "Ġlike": 510,
+      "Ġfe": 511,
+      "Ġfelt": 512,
+      "Ġlooked": 513,
+      "Ġsome": 514,
+      "Ġaround": 515,
+      "Ġsee": 516,
+      "Ġme": 517,
+      "ame": 518,
+      "omet": 519,
+      "ĠW": 520,
+      "Ġro": 521,
+      "Ġbr": 522,
+      "ure": 523,
+      "ĠTimmy": 524,
+      "Ġbir": 525,
+      "get": 526,
+      "ice": 527,
+      "Ġboy": 528,
+      "Ġwould": 529,
+      "Ġstart": 530,
+      "ong": 531,
+      "â": 532,
+      "âĤ": 533,
+      "âĤ¬": 534,
+      "ss": 535,
+      "Ġbird": 536,
+      "Ġsomet": 537,
+      "Ġfa": 538,
+      "dd": 539,
+      "ings": 540,
+      "Ġag": 541,
+      "Ġwor": 542,
+      "Ġwhat": 543,
+      "ade": 544,
+      "Ġmake": 545,
+      "king": 546,
+      ",\"": 547,
+      "ie": 548,
+      "ĠBut": 549,
+      "own": 550,
+      "Ġtre": 551,
+      "Ġran": 552,
+      "gether": 553,
+      "ĠYou": 554,
+      "Ġaway": 555,
+      "ag": 556,
+      "ared": 557,
+      "Ġsays": 558,
+      "Ġstarted": 559,
+      "oud": 560,
+      "if": 561,
+      "Ġmade": 562,
+      "Ġcar": 563,
+      "Ġsomething": 564,
+      "Ġtogether": 565,
+      "ited": 566,
+      "ther": 567,
+      "Ġpark": 568,
+      "Ġco": 569,
+      "Ġsad": 570,
+      "Ġother": 571,
+      "Ġnew": 572,
+      "Ġexc": 573,
+      "Ġput": 574,
+      "Ġfrom": 575,
+      "Ġpr": 576,
+      "ell": 577,
+      "ĠL": 578,
+      "Ġla": 579,
+      "Ġhug": 580,
+      "ble": 581,
+      "Ġmu": 582,
+      "ook": 583,
+      "ach": 584,
+      "Ġagain": 585,
+      "Ġhome": 586,
+      "Ġwhen": 587,
+      "Ġgood": 588,
+      "hat": 589,
+      "ep": 590,
+      "Ġwho": 591,
+      "est": 592,
+      "ried": 593,
+      "Ġfound": 594,
+      "Ġfl": 595,
+      "Ġthen": 596,
+      "Ġch": 597,
+      "Ġdec": 598,
+      "pped": 599,
+      "Ġwal": 600,
+      "as": 601,
+      "Ġget": 602,
+      "qu": 603,
+      "Ġplaying": 604,
+      "ought": 605,
+      "pl": 606,
+      "ĠE": 607,
+      "ally": 608,
+      "Ġsw": 609,
+      "ous": 610,
+      "nn": 611,
+      "Ġthings": 612,
+      "Ġexcited": 613,
+      "Ġliked": 614,
+      "Ġgot": 615,
+      "uck": 616,
+      "ro": 617,
+      "ided": 618,
+      "Ġdecided": 619,
+      "Ġbl": 620,
+      "ave": 621,
+      "Ġcame": 622,
+      "Ġevery": 623,
+      "ara": 624,
+      "Ġyour": 625,
+      "ĠD": 626,
+      "ĠMom": 627,
+      "ax": 628,
+      "Ġv": 629,
+      "Ġscared": 630,
+      "ust": 631,
+      "Ġone": 632,
+      "Ġdog": 633,
+      "Ġdown": 634,
+      "ouse": 635,
+      "ny": 636,
+      "Ġfind": 637,
+      "Ġcare": 638,
+      "Ġbec": 639,
+      "ist": 640,
+      "Ġthan": 641,
+      "Ġfeel": 642,
+      "ap": 643,
+      "ue": 644,
+      "udd": 645,
+      "nna": 646,
+      "Ġdad": 647,
+      "Ġab": 648,
+      "They": 649,
+      "Ġmy": 650,
+      "Ġlook": 651,
+      "Ġke": 652,
+      "our": 653,
+      "Ġwill": 654,
+      "arn": 655,
+      "Ġgr": 656,
+      "us": 657,
+      "nder": 658,
+      "Ġgra": 659,
+      "ways": 660,
+      "ess": 661,
+      "Ġdidn": 662,
+      "Ġpo": 663,
+      "fe": 664,
+      "Ġalways": 665,
+      "Ġtook": 666,
+      "Ġabout": 667,
+      "ant": 668,
+      "bb": 669,
+      "Ġlot": 670,
+      "Ġmommy": 671,
+      "Ġtoys": 672,
+      "Ġkne": 673,
+      "ers": 674,
+      "ise": 675,
+      "ite": 676,
+      "Ġoutside": 677,
+      "Ġtree": 678,
+      "Ġthought": 679,
+      "Ġball": 680,
+      "ĠF": 681,
+      "Ġmore": 682,
+      "eci": 683,
+      "ĠSam": 684,
+      "ened": 685,
+      "ched": 686,
+      "Ġta": 687,
+      "Ġold": 688,
+      "Ġho": 689,
+      "Ġlearn": 690,
+      "ret": 691,
+      "Ġint": 692,
+      "ged": 693,
+      "Ġknow": 694,
+      "Ġlaug": 695,
+      "Ġtake": 696,
+      "ecial": 697,
+      "Ġspecial": 698,
+      "Ġsor": 699,
+      "Ġcat": 700,
+      "udden": 701,
+      "Ġknew": 702,
+      "ĠMax": 703,
+      "ive": 704,
+      "ia": 705,
+      "Ġpe": 706,
+      "Ġany": 707,
+      "uddenly": 708,
+      "Ġtr": 709,
+      "fter": 710,
+      "Ġhow": 711,
+      "ma": 712,
+      "Ġshow": 713,
+      "ink": 714,
+      "Ġsorry": 715,
+      "Ġtw": 716,
+      "But": 717,
+      "Ġmuch": 718,
+      "au": 719,
+      "ish": 720,
+      "Ġrun": 721,
+      "and": 722,
+      "Ġhand": 723,
+      "Ġsl": 724,
+      "ven": 725,
+      "Ġhouse": 726,
+      "Ġor": 727,
+      "Ġop": 728,
+      "Ġsk": 729,
+      "Ġmo": 730,
+      "Ġclo": 731,
+      "Ġsun": 732,
+      "Ġinto": 733,
+      "Ġtried": 734,
+      "Ġinside": 735,
+      "ate": 736,
+      "Ġtold": 737,
+      "Ġwater": 738,
+      "Ġthis": 739,
+      "Ġen": 740,
+      "ank": 741,
+      "Ġif": 742,
+      "Ġproud": 743,
+      "ump": 744,
+      "Ġqu": 745,
+      "Ġover": 746,
+      "Ġcouldn": 747,
+      "Tom": 748,
+      "dy": 749,
+      "Ġgave": 750,
+      "Ġheard": 751,
+      "ĠSara": 752,
+      "Ġnever": 753,
+      "by": 754,
+      "Ġeach": 755,
+      "ace": 756,
+      "Ġroom": 757,
+      "ty": 758,
+      "Ġexpl": 759,
+      "ĠEvery": 760,
+      "Ġeat": 761,
+      "ĠWe": 762,
+      "other": 763,
+      "ause": 764,
+      "Ġpick": 765,
+      "ion": 766,
+      "Ġpret": 767,
+      "ĠAnna": 768,
+      "Mom": 769,
+      "Ġplayed": 770,
+      "ge": 771,
+      "etter": 772,
+      "Ġjust": 773,
+      "ak": 774,
+      "Ġhugged": 775,
+      "Ġgre": 776,
+      "here": 777,
+      "Ġoff": 778,
+      "Ġcom": 779,
+      "Ġwat": 780,
+      "oth": 781,
+      "Ġbox": 782,
+      "Ġnice": 783,
+      "'m": 784,
+      "Ġstr": 785,
+      "Ġbecause": 786,
+      "Ġneed": 787,
+      "ile": 788,
+      "Ġmany": 789,
+      "Ġfo": 790,
+      "uc": 791,
+      "Ġjo": 792,
+      "Ġbear": 793,
+      "Ġsmall": 794,
+      "iz": 795,
+      "Ġbu": 796,
+      "ving": 797,
+      "Ġlong": 798,
+      "ine": 799,
+      "gry": 800,
+      "imal": 801,
+      "ort": 802,
+      "Ġanimal": 803,
+      "Ġsn": 804,
+      "ough": 805,
+      "Ġtry": 806,
+      "Ġunt": 807,
+      "ild": 808,
+      "sel": 809,
+      "urt": 810,
+      "Ġuntil": 811,
+      "ĠJack": 812,
+      "sed": 813,
+      "Ġkind": 814,
+      "Ġlearned": 815,
+      "Ġim": 816,
+      "Ġlove": 817,
+      "Ġbetter": 818,
+      "ft": 819,
+      "Ġflow": 820,
+      "Ġcall": 821,
+      "Ġdon": 822,
+      "He": 823,
+      "ĠJo": 824,
+      "Ġend": 825,
+      "Ġad": 826,
+      "ull": 827,
+      "Ġbest": 828,
+      "vent": 829,
+      "Yes": 830,
+      "ars": 831,
+      "urp": 832,
+      "ady": 833,
+      "em": 834,
+      "ream": 835,
+      "aut": 836,
+      "urn": 837,
+      "ĠMia": 838,
+      "Ġcle": 839,
+      "ĠAnd": 840,
+      "ves": 841,
+      "Åĵ": 842,
+      "arden": 843,
+      "ĠC": 844,
+      "Ġgarden": 845,
+      "Ġfast": 846,
+      "Ġcareful": 847,
+      "ber": 848,
+      "Ġbeaut": 849,
+      "lly": 850,
+      "Th": 851,
+      "Ġeven": 852,
+      "pt": 853,
+      "Ġbra": 854,
+      "ies": 855,
+      "Ġthanked": 856,
+      "Ġfi": 857,
+      "ĠSo": 858,
+      "Ġche": 859,
+      "Ġlaughed": 860,
+      "Ġsky": 861,
+      "Ġby": 862,
+      "It": 863,
+      "ĠHer": 864,
+      "Ġjump": 865,
+      "Ġgl": 866,
+      "ase": 867,
+      "Ġte": 868,
+      "Ġloud": 869,
+      "self": 870,
+      "iny": 871,
+      "Ġway": 872,
+      "wn": 873,
+      "Ġbeauti": 874,
+      "Ġcome": 875,
+      "lew": 876,
+      "Ġlist": 877,
+      "Ġwo": 878,
+      "Ġbeautiful": 879,
+      "Ġhard": 880,
+      "Ġlots": 881,
+      "Ġra": 882,
+      "Ġstill": 883,
+      "Ġfam": 884,
+      "Ġanimals": 885,
+      "Ġâ": 886,
+      "Ġhurt": 887,
+      "Ġunder": 888,
+      "ĠThen": 889,
+      "ect": 890,
+      "Ġstay": 891,
+      "ning": 892,
+      "Ġlet": 893,
+      "ct": 894,
+      "Ġboth": 895,
+      "Ġcu": 896,
+      "ĠHis": 897,
+      "hes": 898,
+      "be": 899,
+      "Ġsafe": 900,
+      "ĠFrom": 901,
+      "ool": 902,
+      "ree": 903,
+      "Ġcol": 904,
+      "Ġtwo": 905,
+      "Ġbad": 906,
+      "Ġbook": 907,
+      "Ġimp": 908,
+      "ople": 909,
+      "Ġrem": 910,
+      "ane": 911,
+      "Ġlived": 912,
+      "kay": 913,
+      "ock": 914,
+      "ucy": 915,
+      "Ġwalked": 916,
+      "Tim": 917,
+      "ast": 918,
+      "urpr": 919,
+      "Ġbrave": 920,
+      "Ġshould": 921,
+      "Ġits": 922,
+      "ob": 923,
+      "ished": 924,
+      "Ġangry": 925,
+      "Ġsurpr": 926,
+      "Ġfamily": 927,
+      "ress": 928,
+      "igh": 929,
+      "When": 930,
+      "Ġpeople": 931,
+      "Ġflew": 932,
+      "Ġcalled": 933,
+      "No": 934,
+      "Ġred": 935,
+      "Ġstor": 936,
+      "ip": 937,
+      "Ġsoon": 938,
+      "Ġfin": 939,
+      "hn": 940,
+      "Let": 941,
+      "led": 942,
+      "Ġkept": 943,
+      "Ġfore": 944,
+      "fore": 945,
+      "Ġdan": 946,
+      "Ġshare": 947,
+      "Ġfly": 948,
+      "xt": 949,
+      "ised": 950,
+      "dded": 951,
+      "Ġpic": 952,
+      "âĤ¬â": 953,
+      "Ġdoor": 954,
+      "Ġgoing": 955,
+      "ĠP": 956,
+      "ĠWhen": 957,
+      "Ġpretty": 958,
+      "Ġkeep": 959,
+      "Ġnow": 960,
+      "Ġrock": 961,
+      "Ġclean": 962,
+      "Ġdra": 963,
+      "ary": 964,
+      "Ġadvent": 965,
+      "ummy": 966,
+      "Ġnoise": 967,
+      "Ġopened": 968,
+      "Ġshiny": 969,
+      "age": 970,
+      "Ġexplore": 971,
+      "Ġwind": 972,
+      "Ġdoll": 973,
+      "Ġcry": 974,
+      "Ġcon": 975,
+      "Ġide": 976,
+      "Ġbefore": 977,
+      "ied": 978,
+      "Ben": 979,
+      "illy": 980,
+      "Ġreal": 981,
+      "Ġsto": 982,
+      "so": 983,
+      "What": 984,
+      "Ġground": 985,
+      "Ġalso": 986,
+      "les": 987,
+      "Ġturn": 988,
+      "Ħ¢": 989,
+      "âĤ¬âĦ¢": 990,
+      "Ġey": 991,
+      "That": 992,
+      "Ġfar": 993,
+      "Ġidea": 994,
+      "Ġfr": 995,
+      "oon": 996,
+      "Ġnodded": 997,
+      "Ġcolor": 998,
+      "Ġwar": 999
+    },
+    "merges": [
+      [
+        "h",
+        "e"
+      ],
+      [
+        "Ġ",
+        "t"
+      ],
+      [
+        "Ġ",
+        "a"
+      ],
+      [
+        "Ġ",
+        "s"
+      ],
+      [
+        "Ġ",
+        "w"
+      ],
+      [
+        "n",
+        "d"
+      ],
+      [
+        "Ġt",
+        "he"
+      ],
+      [
+        "e",
+        "d"
+      ],
+      [
+        "Ġa",
+        "nd"
+      ],
+      [
+        "Ġt",
+        "o"
+      ],
+      [
+        "Ġ",
+        "b"
+      ],
+      [
+        "i",
+        "n"
+      ],
+      [
+        "Ġ",
+        "h"
+      ],
+      [
+        "Ġw",
+        "a"
+      ],
+      [
+        "r",
+        "e"
+      ],
+      [
+        "o",
+        "u"
+      ],
+      [
+        "Ġ",
+        "f"
+      ],
+      [
+        "i",
+        "t"
+      ],
+      [
+        "Ġ",
+        "l"
+      ],
+      [
+        "Ġ",
+        "c"
+      ],
+      [
+        "Ġ",
+        "d"
+      ],
+      [
+        "e",
+        "r"
+      ],
+      [
+        "Ġ",
+        "he"
+      ],
+      [
+        "Ġ",
+        "m"
+      ],
+      [
+        "Ġ",
+        "p"
+      ],
+      [
+        "Ġwa",
+        "s"
+      ],
+      [
+        "o",
+        "m"
+      ],
+      [
+        "Ġ",
+        "T"
+      ],
+      [
+        "Ġ",
+        "o"
+      ],
+      [
+        "a",
+        "y"
+      ],
+      [
+        "a",
+        "r"
+      ],
+      [
+        "i",
+        "s"
+      ],
+      [
+        "in",
+        "g"
+      ],
+      [
+        "i",
+        "l"
+      ],
+      [
+        "Ġ",
+        "g"
+      ],
+      [
+        "i",
+        "d"
+      ],
+      [
+        "a",
+        "t"
+      ],
+      [
+        "e",
+        "n"
+      ],
+      [
+        "Ġ",
+        "n"
+      ],
+      [
+        "Ġs",
+        "a"
+      ],
+      [
+        "Ġh",
+        "a"
+      ],
+      [
+        "Ġ",
+        "S"
+      ],
+      [
+        "ĠT",
+        "he"
+      ],
+      [
+        "a",
+        "n"
+      ],
+      [
+        "i",
+        "m"
+      ],
+      [
+        "o",
+        "r"
+      ],
+      [
+        "o",
+        "n"
+      ],
+      [
+        "Ġ",
+        "it"
+      ],
+      [
+        "Ġt",
+        "h"
+      ],
+      [
+        "l",
+        "l"
+      ],
+      [
+        "l",
+        "e"
+      ],
+      [
+        "Ġ",
+        "H"
+      ],
+      [
+        "Ġhe",
+        "r"
+      ],
+      [
+        "o",
+        "t"
+      ],
+      [
+        "e",
+        "t"
+      ],
+      [
+        "i",
+        "r"
+      ],
+      [
+        "ĠS",
+        "he"
+      ],
+      [
+        "e",
+        "s"
+      ],
+      [
+        "ĠH",
+        "e"
+      ],
+      [
+        "Ġ",
+        "in"
+      ],
+      [
+        "v",
+        "er"
+      ],
+      [
+        "u",
+        "t"
+      ],
+      [
+        "o",
+        "w"
+      ],
+      [
+        "ĠThe",
+        "y"
+      ],
+      [
+        "c",
+        "k"
+      ],
+      [
+        "Ġ",
+        "e"
+      ],
+      [
+        "Ġ",
+        "u"
+      ],
+      [
+        "l",
+        "d"
+      ],
+      [
+        "Ġ",
+        "y"
+      ],
+      [
+        "o",
+        "o"
+      ],
+      [
+        "Ġsa",
+        "id"
+      ],
+      [
+        "i",
+        "g"
+      ],
+      [
+        "Ġ",
+        "\""
+      ],
+      [
+        "il",
+        "y"
+      ],
+      [
+        "Ġ",
+        "r"
+      ],
+      [
+        "Ġb",
+        "e"
+      ],
+      [
+        "a",
+        "m"
+      ],
+      [
+        "Ġs",
+        "t"
+      ],
+      [
+        "Ġ",
+        "I"
+      ],
+      [
+        "c",
+        "e"
+      ],
+      [
+        "k",
+        "e"
+      ],
+      [
+        "Ġs",
+        "he"
+      ],
+      [
+        "v",
+        "e"
+      ],
+      [
+        "p",
+        "p"
+      ],
+      [
+        "it",
+        "h"
+      ],
+      [
+        "Ġw",
+        "ith"
+      ],
+      [
+        "L",
+        "ily"
+      ],
+      [
+        "O",
+        "n"
+      ],
+      [
+        "Ġo",
+        "n"
+      ],
+      [
+        "Ġy",
+        "ou"
+      ],
+      [
+        "Ġo",
+        "f"
+      ],
+      [
+        "k",
+        "ed"
+      ],
+      [
+        "r",
+        "i"
+      ],
+      [
+        "Ġs",
+        "o"
+      ],
+      [
+        "n",
+        "t"
+      ],
+      [
+        "Ġh",
+        "is"
+      ],
+      [
+        "Ġp",
+        "l"
+      ],
+      [
+        "a",
+        "d"
+      ],
+      [
+        "ver",
+        "y"
+      ],
+      [
+        "Ġd",
+        "ay"
+      ],
+      [
+        "Ġth",
+        "at"
+      ],
+      [
+        "Ġu",
+        "p"
+      ],
+      [
+        "Ġpl",
+        "ay"
+      ],
+      [
+        "Ġha",
+        "d"
+      ],
+      [
+        "s",
+        "t"
+      ],
+      [
+        "Ġthe",
+        "y"
+      ],
+      [
+        "Ġw",
+        "e"
+      ],
+      [
+        "Ġ",
+        "Lily"
+      ],
+      [
+        "Ġm",
+        "om"
+      ],
+      [
+        "Ġf",
+        "or"
+      ],
+      [
+        "Ġ",
+        "B"
+      ],
+      [
+        "e",
+        "l"
+      ],
+      [
+        "u",
+        "n"
+      ],
+      [
+        "ou",
+        "ld"
+      ],
+      [
+        "'",
+        "s"
+      ],
+      [
+        "m",
+        "y"
+      ],
+      [
+        "T",
+        "he"
+      ],
+      [
+        "Ġl",
+        "i"
+      ],
+      [
+        "c",
+        "h"
+      ],
+      [
+        "en",
+        "t"
+      ],
+      [
+        "Ġha",
+        "pp"
+      ],
+      [
+        "Ġwa",
+        "nt"
+      ],
+      [
+        "it",
+        "t"
+      ],
+      [
+        "Ġs",
+        "h"
+      ],
+      [
+        "ou",
+        "t"
+      ],
+      [
+        "he",
+        "r"
+      ],
+      [
+        "Ġd",
+        "o"
+      ],
+      [
+        "l",
+        "y"
+      ],
+      [
+        "Ġn",
+        "ot"
+      ],
+      [
+        "om",
+        "e"
+      ],
+      [
+        "s",
+        "e"
+      ],
+      [
+        "Ġ",
+        "very"
+      ],
+      [
+        "ou",
+        "nd"
+      ],
+      [
+        "e",
+        "nd"
+      ],
+      [
+        "a",
+        "ll"
+      ],
+      [
+        "Ġw",
+        "h"
+      ],
+      [
+        "im",
+        "e"
+      ],
+      [
+        "itt",
+        "le"
+      ],
+      [
+        "Ġ",
+        "k"
+      ],
+      [
+        "Ġt",
+        "ime"
+      ],
+      [
+        "Ġl",
+        "ittle"
+      ],
+      [
+        "a",
+        "l"
+      ],
+      [
+        "Ġ",
+        "Ċ"
+      ],
+      [
+        "Ġthe",
+        "re"
+      ],
+      [
+        "Ġn",
+        "e"
+      ],
+      [
+        "h",
+        "t"
+      ],
+      [
+        "Ġ",
+        "M"
+      ],
+      [
+        "Ġ",
+        "re"
+      ],
+      [
+        "Ġhapp",
+        "y"
+      ],
+      [
+        "Ġs",
+        "m"
+      ],
+      [
+        "Ġb",
+        "ig"
+      ],
+      [
+        "Ġ",
+        "is"
+      ],
+      [
+        "ĠI",
+        "t"
+      ],
+      [
+        "Ġb",
+        "ut"
+      ],
+      [
+        "a",
+        "ck"
+      ],
+      [
+        "Ġa",
+        "n"
+      ],
+      [
+        "Ġsa",
+        "w"
+      ],
+      [
+        "ri",
+        "end"
+      ],
+      [
+        ".",
+        "\""
+      ],
+      [
+        "Ġf",
+        "riend"
+      ],
+      [
+        "r",
+        "y"
+      ],
+      [
+        "Ġa",
+        "s"
+      ],
+      [
+        "r",
+        "a"
+      ],
+      [
+        "On",
+        "e"
+      ],
+      [
+        "id",
+        "e"
+      ],
+      [
+        "a",
+        "ke"
+      ],
+      [
+        "'",
+        "t"
+      ],
+      [
+        "!",
+        "\""
+      ],
+      [
+        "u",
+        "g"
+      ],
+      [
+        "Ġl",
+        "oo"
+      ],
+      [
+        "v",
+        "ed"
+      ],
+      [
+        "t",
+        "er"
+      ],
+      [
+        "Ġwe",
+        "re"
+      ],
+      [
+        "o",
+        "re"
+      ],
+      [
+        "On",
+        "ce"
+      ],
+      [
+        "Ġl",
+        "o"
+      ],
+      [
+        "e",
+        "c"
+      ],
+      [
+        "Ġh",
+        "im"
+      ],
+      [
+        "Ġto",
+        "o"
+      ],
+      [
+        "Ġg",
+        "o"
+      ],
+      [
+        "Ġb",
+        "o"
+      ],
+      [
+        "Ġs",
+        "e"
+      ],
+      [
+        "ir",
+        "l"
+      ],
+      [
+        "Ġa",
+        "re"
+      ],
+      [
+        "Ġ",
+        "j"
+      ],
+      [
+        "ĠT",
+        "im"
+      ],
+      [
+        "ar",
+        "d"
+      ],
+      [
+        "Ġwant",
+        "ed"
+      ],
+      [
+        "il",
+        "l"
+      ],
+      [
+        "Ġg",
+        "irl"
+      ],
+      [
+        "Ġup",
+        "on"
+      ],
+      [
+        "Ġthe",
+        "m"
+      ],
+      [
+        "Ġ",
+        "out"
+      ],
+      [
+        "Ġa",
+        "t"
+      ],
+      [
+        "u",
+        "r"
+      ],
+      [
+        "i",
+        "nd"
+      ],
+      [
+        "Ġthe",
+        "ir"
+      ],
+      [
+        "f",
+        "u"
+      ],
+      [
+        "Ġs",
+        "p"
+      ],
+      [
+        "w",
+        "ay"
+      ],
+      [
+        "Ġ",
+        "A"
+      ],
+      [
+        "Ġsm",
+        "il"
+      ],
+      [
+        "Ġd",
+        "id"
+      ],
+      [
+        "Ġc",
+        "ould"
+      ],
+      [
+        "he",
+        "n"
+      ],
+      [
+        "Ġha",
+        "ve"
+      ],
+      [
+        "a",
+        "in"
+      ],
+      [
+        "ar",
+        "t"
+      ],
+      [
+        "Ġe",
+        "x"
+      ],
+      [
+        "he",
+        "d"
+      ],
+      [
+        "r",
+        "om"
+      ],
+      [
+        "Ġw",
+        "ent"
+      ],
+      [
+        "Ġa",
+        "r"
+      ],
+      [
+        "i",
+        "c"
+      ],
+      [
+        "Ġc",
+        "an"
+      ],
+      [
+        "Ġ",
+        "J"
+      ],
+      [
+        "l",
+        "p"
+      ],
+      [
+        "fu",
+        "l"
+      ],
+      [
+        "ĠB",
+        "en"
+      ],
+      [
+        "oo",
+        "d"
+      ],
+      [
+        "?",
+        "\""
+      ],
+      [
+        "h",
+        "ing"
+      ],
+      [
+        "Ġfriend",
+        "s"
+      ],
+      [
+        "Ġn",
+        "am"
+      ],
+      [
+        "ig",
+        "ht"
+      ],
+      [
+        "Ġa",
+        "ll"
+      ],
+      [
+        "Ġhe",
+        "lp"
+      ],
+      [
+        "Ġk",
+        "n"
+      ],
+      [
+        "u",
+        "m"
+      ],
+      [
+        "on",
+        "e"
+      ],
+      [
+        "ar",
+        "k"
+      ],
+      [
+        "Ġb",
+        "ack"
+      ],
+      [
+        "Ġs",
+        "ay"
+      ],
+      [
+        "Y",
+        "ou"
+      ],
+      [
+        "Ġf",
+        "un"
+      ],
+      [
+        "Ġc",
+        "l"
+      ],
+      [
+        "ĠT",
+        "om"
+      ],
+      [
+        "a",
+        "re"
+      ],
+      [
+        "Ġn",
+        "o"
+      ],
+      [
+        "Ġl",
+        "e"
+      ],
+      [
+        "o",
+        "p"
+      ],
+      [
+        "Ġsmil",
+        "ed"
+      ],
+      [
+        "Ġs",
+        "c"
+      ],
+      [
+        "Ġnam",
+        "ed"
+      ],
+      [
+        "Ġa",
+        "l"
+      ],
+      [
+        "ou",
+        "g"
+      ],
+      [
+        "Ġlo",
+        "ved"
+      ],
+      [
+        "s",
+        "ide"
+      ],
+      [
+        "el",
+        "t"
+      ],
+      [
+        "Ġ",
+        "One"
+      ],
+      [
+        "Ġm",
+        "an"
+      ],
+      [
+        "Ġas",
+        "ked"
+      ],
+      [
+        "i",
+        "ck"
+      ],
+      [
+        "Ġto",
+        "y"
+      ],
+      [
+        "Ġli",
+        "ke"
+      ],
+      [
+        "Ġf",
+        "e"
+      ],
+      [
+        "Ġf",
+        "elt"
+      ],
+      [
+        "Ġloo",
+        "ked"
+      ],
+      [
+        "Ġs",
+        "ome"
+      ],
+      [
+        "Ġar",
+        "ound"
+      ],
+      [
+        "Ġse",
+        "e"
+      ],
+      [
+        "Ġm",
+        "e"
+      ],
+      [
+        "am",
+        "e"
+      ],
+      [
+        "om",
+        "et"
+      ],
+      [
+        "Ġ",
+        "W"
+      ],
+      [
+        "Ġr",
+        "o"
+      ],
+      [
+        "Ġb",
+        "r"
+      ],
+      [
+        "u",
+        "re"
+      ],
+      [
+        "ĠTim",
+        "my"
+      ],
+      [
+        "Ġb",
+        "ir"
+      ],
+      [
+        "g",
+        "et"
+      ],
+      [
+        "i",
+        "ce"
+      ],
+      [
+        "Ġbo",
+        "y"
+      ],
+      [
+        "Ġw",
+        "ould"
+      ],
+      [
+        "Ġst",
+        "art"
+      ],
+      [
+        "on",
+        "g"
+      ],
+      [
+        "Ã",
+        "¢"
+      ],
+      [
+        "â",
+        "Ĥ"
+      ],
+      [
+        "âĤ",
+        "¬"
+      ],
+      [
+        "s",
+        "s"
+      ],
+      [
+        "Ġbir",
+        "d"
+      ],
+      [
+        "Ġs",
+        "omet"
+      ],
+      [
+        "Ġf",
+        "a"
+      ],
+      [
+        "d",
+        "d"
+      ],
+      [
+        "ing",
+        "s"
+      ],
+      [
+        "Ġa",
+        "g"
+      ],
+      [
+        "Ġw",
+        "or"
+      ],
+      [
+        "Ġwh",
+        "at"
+      ],
+      [
+        "ad",
+        "e"
+      ],
+      [
+        "Ġm",
+        "ake"
+      ],
+      [
+        "k",
+        "ing"
+      ],
+      [
+        ",",
+        "\""
+      ],
+      [
+        "i",
+        "e"
+      ],
+      [
+        "ĠB",
+        "ut"
+      ],
+      [
+        "ow",
+        "n"
+      ],
+      [
+        "Ġt",
+        "re"
+      ],
+      [
+        "Ġr",
+        "an"
+      ],
+      [
+        "get",
+        "her"
+      ],
+      [
+        "Ġ",
+        "You"
+      ],
+      [
+        "Ġa",
+        "way"
+      ],
+      [
+        "a",
+        "g"
+      ],
+      [
+        "ar",
+        "ed"
+      ],
+      [
+        "Ġsay",
+        "s"
+      ],
+      [
+        "Ġstart",
+        "ed"
+      ],
+      [
+        "ou",
+        "d"
+      ],
+      [
+        "i",
+        "f"
+      ],
+      [
+        "Ġm",
+        "ade"
+      ],
+      [
+        "Ġc",
+        "ar"
+      ],
+      [
+        "Ġsomet",
+        "hing"
+      ],
+      [
+        "Ġto",
+        "gether"
+      ],
+      [
+        "it",
+        "ed"
+      ],
+      [
+        "t",
+        "her"
+      ],
+      [
+        "Ġp",
+        "ark"
+      ],
+      [
+        "Ġc",
+        "o"
+      ],
+      [
+        "Ġsa",
+        "d"
+      ],
+      [
+        "Ġo",
+        "ther"
+      ],
+      [
+        "Ġne",
+        "w"
+      ],
+      [
+        "Ġex",
+        "c"
+      ],
+      [
+        "Ġp",
+        "ut"
+      ],
+      [
+        "Ġf",
+        "rom"
+      ],
+      [
+        "Ġp",
+        "r"
+      ],
+      [
+        "e",
+        "ll"
+      ],
+      [
+        "Ġ",
+        "L"
+      ],
+      [
+        "Ġl",
+        "a"
+      ],
+      [
+        "Ġh",
+        "ug"
+      ],
+      [
+        "b",
+        "le"
+      ],
+      [
+        "Ġm",
+        "u"
+      ],
+      [
+        "oo",
+        "k"
+      ],
+      [
+        "a",
+        "ch"
+      ],
+      [
+        "Ġag",
+        "ain"
+      ],
+      [
+        "Ġh",
+        "ome"
+      ],
+      [
+        "Ġw",
+        "hen"
+      ],
+      [
+        "Ġg",
+        "ood"
+      ],
+      [
+        "h",
+        "at"
+      ],
+      [
+        "e",
+        "p"
+      ],
+      [
+        "Ġwh",
+        "o"
+      ],
+      [
+        "es",
+        "t"
+      ],
+      [
+        "ri",
+        "ed"
+      ],
+      [
+        "Ġf",
+        "ound"
+      ],
+      [
+        "Ġf",
+        "l"
+      ],
+      [
+        "Ġthe",
+        "n"
+      ],
+      [
+        "Ġc",
+        "h"
+      ],
+      [
+        "Ġd",
+        "ec"
+      ],
+      [
+        "pp",
+        "ed"
+      ],
+      [
+        "Ġwa",
+        "l"
+      ],
+      [
+        "a",
+        "s"
+      ],
+      [
+        "Ġg",
+        "et"
+      ],
+      [
+        "q",
+        "u"
+      ],
+      [
+        "Ġplay",
+        "ing"
+      ],
+      [
+        "oug",
+        "ht"
+      ],
+      [
+        "p",
+        "l"
+      ],
+      [
+        "Ġ",
+        "E"
+      ],
+      [
+        "all",
+        "y"
+      ],
+      [
+        "Ġs",
+        "w"
+      ],
+      [
+        "ou",
+        "s"
+      ],
+      [
+        "n",
+        "n"
+      ],
+      [
+        "Ġth",
+        "ings"
+      ],
+      [
+        "Ġexc",
+        "ited"
+      ],
+      [
+        "Ġli",
+        "ked"
+      ],
+      [
+        "Ġg",
+        "ot"
+      ],
+      [
+        "u",
+        "ck"
+      ],
+      [
+        "r",
+        "o"
+      ],
+      [
+        "id",
+        "ed"
+      ],
+      [
+        "Ġdec",
+        "ided"
+      ],
+      [
+        "Ġb",
+        "l"
+      ],
+      [
+        "a",
+        "ve"
+      ],
+      [
+        "Ġc",
+        "ame"
+      ],
+      [
+        "Ġe",
+        "very"
+      ],
+      [
+        "ar",
+        "a"
+      ],
+      [
+        "Ġyou",
+        "r"
+      ],
+      [
+        "Ġ",
+        "D"
+      ],
+      [
+        "ĠM",
+        "om"
+      ],
+      [
+        "a",
+        "x"
+      ],
+      [
+        "Ġ",
+        "v"
+      ],
+      [
+        "Ġsc",
+        "ared"
+      ],
+      [
+        "u",
+        "st"
+      ],
+      [
+        "Ġon",
+        "e"
+      ],
+      [
+        "Ġdo",
+        "g"
+      ],
+      [
+        "Ġd",
+        "own"
+      ],
+      [
+        "ou",
+        "se"
+      ],
+      [
+        "n",
+        "y"
+      ],
+      [
+        "Ġf",
+        "ind"
+      ],
+      [
+        "Ġc",
+        "are"
+      ],
+      [
+        "Ġbe",
+        "c"
+      ],
+      [
+        "is",
+        "t"
+      ],
+      [
+        "Ġth",
+        "an"
+      ],
+      [
+        "Ġfe",
+        "el"
+      ],
+      [
+        "a",
+        "p"
+      ],
+      [
+        "u",
+        "e"
+      ],
+      [
+        "u",
+        "dd"
+      ],
+      [
+        "nn",
+        "a"
+      ],
+      [
+        "Ġd",
+        "ad"
+      ],
+      [
+        "Ġa",
+        "b"
+      ],
+      [
+        "The",
+        "y"
+      ],
+      [
+        "Ġm",
+        "y"
+      ],
+      [
+        "Ġloo",
+        "k"
+      ],
+      [
+        "Ġ",
+        "ke"
+      ],
+      [
+        "ou",
+        "r"
+      ],
+      [
+        "Ġw",
+        "ill"
+      ],
+      [
+        "ar",
+        "n"
+      ],
+      [
+        "Ġg",
+        "r"
+      ],
+      [
+        "u",
+        "s"
+      ],
+      [
+        "nd",
+        "er"
+      ],
+      [
+        "Ġg",
+        "ra"
+      ],
+      [
+        "way",
+        "s"
+      ],
+      [
+        "es",
+        "s"
+      ],
+      [
+        "Ġdid",
+        "n"
+      ],
+      [
+        "Ġp",
+        "o"
+      ],
+      [
+        "f",
+        "e"
+      ],
+      [
+        "Ġal",
+        "ways"
+      ],
+      [
+        "Ġtoo",
+        "k"
+      ],
+      [
+        "Ġab",
+        "out"
+      ],
+      [
+        "an",
+        "t"
+      ],
+      [
+        "b",
+        "b"
+      ],
+      [
+        "Ġl",
+        "ot"
+      ],
+      [
+        "Ġmom",
+        "my"
+      ],
+      [
+        "Ġtoy",
+        "s"
+      ],
+      [
+        "Ġkn",
+        "e"
+      ],
+      [
+        "er",
+        "s"
+      ],
+      [
+        "is",
+        "e"
+      ],
+      [
+        "it",
+        "e"
+      ],
+      [
+        "Ġout",
+        "side"
+      ],
+      [
+        "Ġtre",
+        "e"
+      ],
+      [
+        "Ġth",
+        "ought"
+      ],
+      [
+        "Ġb",
+        "all"
+      ],
+      [
+        "Ġ",
+        "F"
+      ],
+      [
+        "Ġm",
+        "ore"
+      ],
+      [
+        "ec",
+        "i"
+      ],
+      [
+        "ĠS",
+        "am"
+      ],
+      [
+        "en",
+        "ed"
+      ],
+      [
+        "c",
+        "hed"
+      ],
+      [
+        "Ġt",
+        "a"
+      ],
+      [
+        "Ġo",
+        "ld"
+      ],
+      [
+        "Ġh",
+        "o"
+      ],
+      [
+        "Ġle",
+        "arn"
+      ],
+      [
+        "re",
+        "t"
+      ],
+      [
+        "Ġin",
+        "t"
+      ],
+      [
+        "g",
+        "ed"
+      ],
+      [
+        "Ġkn",
+        "ow"
+      ],
+      [
+        "Ġla",
+        "ug"
+      ],
+      [
+        "Ġt",
+        "ake"
+      ],
+      [
+        "eci",
+        "al"
+      ],
+      [
+        "Ġsp",
+        "ecial"
+      ],
+      [
+        "Ġs",
+        "or"
+      ],
+      [
+        "Ġc",
+        "at"
+      ],
+      [
+        "udd",
+        "en"
+      ],
+      [
+        "Ġkne",
+        "w"
+      ],
+      [
+        "ĠM",
+        "ax"
+      ],
+      [
+        "i",
+        "ve"
+      ],
+      [
+        "i",
+        "a"
+      ],
+      [
+        "Ġp",
+        "e"
+      ],
+      [
+        "Ġan",
+        "y"
+      ],
+      [
+        "udden",
+        "ly"
+      ],
+      [
+        "Ġt",
+        "r"
+      ],
+      [
+        "f",
+        "ter"
+      ],
+      [
+        "Ġh",
+        "ow"
+      ],
+      [
+        "m",
+        "a"
+      ],
+      [
+        "Ġsh",
+        "ow"
+      ],
+      [
+        "in",
+        "k"
+      ],
+      [
+        "Ġsor",
+        "ry"
+      ],
+      [
+        "Ġt",
+        "w"
+      ],
+      [
+        "B",
+        "ut"
+      ],
+      [
+        "Ġmu",
+        "ch"
+      ],
+      [
+        "a",
+        "u"
+      ],
+      [
+        "is",
+        "h"
+      ],
+      [
+        "Ġr",
+        "un"
+      ],
+      [
+        "a",
+        "nd"
+      ],
+      [
+        "Ġha",
+        "nd"
+      ],
+      [
+        "Ġs",
+        "l"
+      ],
+      [
+        "v",
+        "en"
+      ],
+      [
+        "Ġh",
+        "ouse"
+      ],
+      [
+        "Ġo",
+        "r"
+      ],
+      [
+        "Ġo",
+        "p"
+      ],
+      [
+        "Ġs",
+        "k"
+      ],
+      [
+        "Ġm",
+        "o"
+      ],
+      [
+        "Ġcl",
+        "o"
+      ],
+      [
+        "Ġs",
+        "un"
+      ],
+      [
+        "Ġint",
+        "o"
+      ],
+      [
+        "Ġt",
+        "ried"
+      ],
+      [
+        "Ġin",
+        "side"
+      ],
+      [
+        "at",
+        "e"
+      ],
+      [
+        "Ġto",
+        "ld"
+      ],
+      [
+        "Ġwa",
+        "ter"
+      ],
+      [
+        "Ġth",
+        "is"
+      ],
+      [
+        "Ġ",
+        "en"
+      ],
+      [
+        "an",
+        "k"
+      ],
+      [
+        "Ġ",
+        "if"
+      ],
+      [
+        "Ġpr",
+        "oud"
+      ],
+      [
+        "um",
+        "p"
+      ],
+      [
+        "Ġ",
+        "qu"
+      ],
+      [
+        "Ġo",
+        "ver"
+      ],
+      [
+        "Ġcould",
+        "n"
+      ],
+      [
+        "T",
+        "om"
+      ],
+      [
+        "d",
+        "y"
+      ],
+      [
+        "Ġg",
+        "ave"
+      ],
+      [
+        "Ġhe",
+        "ard"
+      ],
+      [
+        "ĠS",
+        "ara"
+      ],
+      [
+        "Ġne",
+        "ver"
+      ],
+      [
+        "b",
+        "y"
+      ],
+      [
+        "Ġe",
+        "ach"
+      ],
+      [
+        "a",
+        "ce"
+      ],
+      [
+        "Ġro",
+        "om"
+      ],
+      [
+        "t",
+        "y"
+      ],
+      [
+        "Ġex",
+        "pl"
+      ],
+      [
+        "ĠE",
+        "very"
+      ],
+      [
+        "Ġe",
+        "at"
+      ],
+      [
+        "ĠW",
+        "e"
+      ],
+      [
+        "ot",
+        "her"
+      ],
+      [
+        "au",
+        "se"
+      ],
+      [
+        "Ġp",
+        "ick"
+      ],
+      [
+        "i",
+        "on"
+      ],
+      [
+        "Ġp",
+        "ret"
+      ],
+      [
+        "ĠA",
+        "nna"
+      ],
+      [
+        "M",
+        "om"
+      ],
+      [
+        "Ġplay",
+        "ed"
+      ],
+      [
+        "g",
+        "e"
+      ],
+      [
+        "et",
+        "ter"
+      ],
+      [
+        "Ġj",
+        "ust"
+      ],
+      [
+        "a",
+        "k"
+      ],
+      [
+        "Ġhug",
+        "ged"
+      ],
+      [
+        "Ġg",
+        "re"
+      ],
+      [
+        "he",
+        "re"
+      ],
+      [
+        "Ġof",
+        "f"
+      ],
+      [
+        "Ġc",
+        "om"
+      ],
+      [
+        "Ġwa",
+        "t"
+      ],
+      [
+        "ot",
+        "h"
+      ],
+      [
+        "Ġbo",
+        "x"
+      ],
+      [
+        "Ġn",
+        "ice"
+      ],
+      [
+        "'",
+        "m"
+      ],
+      [
+        "Ġst",
+        "r"
+      ],
+      [
+        "Ġbec",
+        "ause"
+      ],
+      [
+        "Ġne",
+        "ed"
+      ],
+      [
+        "il",
+        "e"
+      ],
+      [
+        "Ġman",
+        "y"
+      ],
+      [
+        "Ġf",
+        "o"
+      ],
+      [
+        "u",
+        "c"
+      ],
+      [
+        "Ġj",
+        "o"
+      ],
+      [
+        "Ġbe",
+        "ar"
+      ],
+      [
+        "Ġsm",
+        "all"
+      ],
+      [
+        "i",
+        "z"
+      ],
+      [
+        "Ġb",
+        "u"
+      ],
+      [
+        "v",
+        "ing"
+      ],
+      [
+        "Ġl",
+        "ong"
+      ],
+      [
+        "in",
+        "e"
+      ],
+      [
+        "g",
+        "ry"
+      ],
+      [
+        "im",
+        "al"
+      ],
+      [
+        "or",
+        "t"
+      ],
+      [
+        "Ġan",
+        "imal"
+      ],
+      [
+        "Ġs",
+        "n"
+      ],
+      [
+        "oug",
+        "h"
+      ],
+      [
+        "Ġt",
+        "ry"
+      ],
+      [
+        "Ġu",
+        "nt"
+      ],
+      [
+        "il",
+        "d"
+      ],
+      [
+        "s",
+        "el"
+      ],
+      [
+        "ur",
+        "t"
+      ],
+      [
+        "Ġunt",
+        "il"
+      ],
+      [
+        "ĠJ",
+        "ack"
+      ],
+      [
+        "s",
+        "ed"
+      ],
+      [
+        "Ġk",
+        "ind"
+      ],
+      [
+        "Ġlearn",
+        "ed"
+      ],
+      [
+        "Ġ",
+        "im"
+      ],
+      [
+        "Ġlo",
+        "ve"
+      ],
+      [
+        "Ġb",
+        "etter"
+      ],
+      [
+        "f",
+        "t"
+      ],
+      [
+        "Ġfl",
+        "ow"
+      ],
+      [
+        "Ġc",
+        "all"
+      ],
+      [
+        "Ġd",
+        "on"
+      ],
+      [
+        "H",
+        "e"
+      ],
+      [
+        "ĠJ",
+        "o"
+      ],
+      [
+        "Ġe",
+        "nd"
+      ],
+      [
+        "Ġa",
+        "d"
+      ],
+      [
+        "u",
+        "ll"
+      ],
+      [
+        "Ġb",
+        "est"
+      ],
+      [
+        "v",
+        "ent"
+      ],
+      [
+        "Y",
+        "es"
+      ],
+      [
+        "ar",
+        "s"
+      ],
+      [
+        "ur",
+        "p"
+      ],
+      [
+        "ad",
+        "y"
+      ],
+      [
+        "e",
+        "m"
+      ],
+      [
+        "re",
+        "am"
+      ],
+      [
+        "a",
+        "ut"
+      ],
+      [
+        "ur",
+        "n"
+      ],
+      [
+        "ĠM",
+        "ia"
+      ],
+      [
+        "Ġc",
+        "le"
+      ],
+      [
+        "ĠA",
+        "nd"
+      ],
+      [
+        "v",
+        "es"
+      ],
+      [
+        "Å",
+        "ĵ"
+      ],
+      [
+        "ard",
+        "en"
+      ],
+      [
+        "Ġ",
+        "C"
+      ],
+      [
+        "Ġg",
+        "arden"
+      ],
+      [
+        "Ġfa",
+        "st"
+      ],
+      [
+        "Ġcare",
+        "ful"
+      ],
+      [
+        "b",
+        "er"
+      ],
+      [
+        "Ġbe",
+        "aut"
+      ],
+      [
+        "ll",
+        "y"
+      ],
+      [
+        "T",
+        "h"
+      ],
+      [
+        "Ġe",
+        "ven"
+      ],
+      [
+        "p",
+        "t"
+      ],
+      [
+        "Ġb",
+        "ra"
+      ],
+      [
+        "i",
+        "es"
+      ],
+      [
+        "Ġthan",
+        "ked"
+      ],
+      [
+        "Ġf",
+        "i"
+      ],
+      [
+        "ĠS",
+        "o"
+      ],
+      [
+        "Ġc",
+        "he"
+      ],
+      [
+        "Ġlaug",
+        "hed"
+      ],
+      [
+        "Ġsk",
+        "y"
+      ],
+      [
+        "Ġb",
+        "y"
+      ],
+      [
+        "I",
+        "t"
+      ],
+      [
+        "ĠH",
+        "er"
+      ],
+      [
+        "Ġj",
+        "ump"
+      ],
+      [
+        "Ġg",
+        "l"
+      ],
+      [
+        "a",
+        "se"
+      ],
+      [
+        "Ġt",
+        "e"
+      ],
+      [
+        "Ġl",
+        "oud"
+      ],
+      [
+        "sel",
+        "f"
+      ],
+      [
+        "in",
+        "y"
+      ],
+      [
+        "Ġwa",
+        "y"
+      ],
+      [
+        "w",
+        "n"
+      ],
+      [
+        "Ġbeaut",
+        "i"
+      ],
+      [
+        "Ġc",
+        "ome"
+      ],
+      [
+        "le",
+        "w"
+      ],
+      [
+        "Ġl",
+        "ist"
+      ],
+      [
+        "Ġw",
+        "o"
+      ],
+      [
+        "Ġbeauti",
+        "ful"
+      ],
+      [
+        "Ġh",
+        "ard"
+      ],
+      [
+        "Ġlot",
+        "s"
+      ],
+      [
+        "Ġr",
+        "a"
+      ],
+      [
+        "Ġst",
+        "ill"
+      ],
+      [
+        "Ġf",
+        "am"
+      ],
+      [
+        "Ġanimal",
+        "s"
+      ],
+      [
+        "Ġ",
+        "â"
+      ],
+      [
+        "Ġh",
+        "urt"
+      ],
+      [
+        "Ġu",
+        "nder"
+      ],
+      [
+        "ĠThe",
+        "n"
+      ],
+      [
+        "ec",
+        "t"
+      ],
+      [
+        "Ġst",
+        "ay"
+      ],
+      [
+        "n",
+        "ing"
+      ],
+      [
+        "Ġl",
+        "et"
+      ],
+      [
+        "c",
+        "t"
+      ],
+      [
+        "Ġb",
+        "oth"
+      ],
+      [
+        "Ġc",
+        "u"
+      ],
+      [
+        "ĠH",
+        "is"
+      ],
+      [
+        "he",
+        "s"
+      ],
+      [
+        "b",
+        "e"
+      ],
+      [
+        "Ġsa",
+        "fe"
+      ],
+      [
+        "ĠF",
+        "rom"
+      ],
+      [
+        "oo",
+        "l"
+      ],
+      [
+        "re",
+        "e"
+      ],
+      [
+        "Ġco",
+        "l"
+      ],
+      [
+        "Ġtw",
+        "o"
+      ],
+      [
+        "Ġb",
+        "ad"
+      ],
+      [
+        "Ġb",
+        "ook"
+      ],
+      [
+        "Ġim",
+        "p"
+      ],
+      [
+        "op",
+        "le"
+      ],
+      [
+        "Ġre",
+        "m"
+      ],
+      [
+        "an",
+        "e"
+      ],
+      [
+        "Ġli",
+        "ved"
+      ],
+      [
+        "k",
+        "ay"
+      ],
+      [
+        "o",
+        "ck"
+      ],
+      [
+        "uc",
+        "y"
+      ],
+      [
+        "Ġwal",
+        "ked"
+      ],
+      [
+        "T",
+        "im"
+      ],
+      [
+        "a",
+        "st"
+      ],
+      [
+        "urp",
+        "r"
+      ],
+      [
+        "Ġbra",
+        "ve"
+      ],
+      [
+        "Ġsh",
+        "ould"
+      ],
+      [
+        "Ġit",
+        "s"
+      ],
+      [
+        "o",
+        "b"
+      ],
+      [
+        "is",
+        "hed"
+      ],
+      [
+        "Ġan",
+        "gry"
+      ],
+      [
+        "Ġs",
+        "urpr"
+      ],
+      [
+        "Ġfam",
+        "ily"
+      ],
+      [
+        "re",
+        "ss"
+      ],
+      [
+        "ig",
+        "h"
+      ],
+      [
+        "W",
+        "hen"
+      ],
+      [
+        "Ġpe",
+        "ople"
+      ],
+      [
+        "Ġf",
+        "lew"
+      ],
+      [
+        "Ġcall",
+        "ed"
+      ],
+      [
+        "N",
+        "o"
+      ],
+      [
+        "Ġr",
+        "ed"
+      ],
+      [
+        "Ġst",
+        "or"
+      ],
+      [
+        "i",
+        "p"
+      ],
+      [
+        "Ġso",
+        "on"
+      ],
+      [
+        "Ġf",
+        "in"
+      ],
+      [
+        "h",
+        "n"
+      ],
+      [
+        "L",
+        "et"
+      ],
+      [
+        "l",
+        "ed"
+      ],
+      [
+        "Ġke",
+        "pt"
+      ],
+      [
+        "Ġf",
+        "ore"
+      ],
+      [
+        "f",
+        "ore"
+      ],
+      [
+        "Ġd",
+        "an"
+      ],
+      [
+        "Ġsh",
+        "are"
+      ],
+      [
+        "Ġf",
+        "ly"
+      ],
+      [
+        "x",
+        "t"
+      ],
+      [
+        "is",
+        "ed"
+      ],
+      [
+        "dd",
+        "ed"
+      ],
+      [
+        "Ġp",
+        "ic"
+      ],
+      [
+        "âĤ¬",
+        "â"
+      ],
+      [
+        "Ġdo",
+        "or"
+      ],
+      [
+        "Ġgo",
+        "ing"
+      ],
+      [
+        "Ġ",
+        "P"
+      ],
+      [
+        "ĠW",
+        "hen"
+      ],
+      [
+        "Ġpret",
+        "ty"
+      ],
+      [
+        "Ġke",
+        "ep"
+      ],
+      [
+        "Ġn",
+        "ow"
+      ],
+      [
+        "Ġro",
+        "ck"
+      ],
+      [
+        "Ġcle",
+        "an"
+      ],
+      [
+        "Ġd",
+        "ra"
+      ],
+      [
+        "ar",
+        "y"
+      ],
+      [
+        "Ġad",
+        "vent"
+      ],
+      [
+        "um",
+        "my"
+      ],
+      [
+        "Ġno",
+        "ise"
+      ],
+      [
+        "Ġop",
+        "ened"
+      ],
+      [
+        "Ġsh",
+        "iny"
+      ],
+      [
+        "ag",
+        "e"
+      ],
+      [
+        "Ġexpl",
+        "ore"
+      ],
+      [
+        "Ġw",
+        "ind"
+      ],
+      [
+        "Ġdo",
+        "ll"
+      ],
+      [
+        "Ġc",
+        "ry"
+      ],
+      [
+        "Ġc",
+        "on"
+      ],
+      [
+        "Ġ",
+        "ide"
+      ],
+      [
+        "Ġbe",
+        "fore"
+      ],
+      [
+        "i",
+        "ed"
+      ],
+      [
+        "B",
+        "en"
+      ],
+      [
+        "il",
+        "ly"
+      ],
+      [
+        "Ġre",
+        "al"
+      ],
+      [
+        "Ġst",
+        "o"
+      ],
+      [
+        "s",
+        "o"
+      ],
+      [
+        "W",
+        "hat"
+      ],
+      [
+        "Ġgr",
+        "ound"
+      ],
+      [
+        "Ġal",
+        "so"
+      ],
+      [
+        "le",
+        "s"
+      ],
+      [
+        "Ġt",
+        "urn"
+      ],
+      [
+        "Ħ",
+        "¢"
+      ],
+      [
+        "âĤ¬â",
+        "Ħ¢"
+      ],
+      [
+        "Ġe",
+        "y"
+      ],
+      [
+        "T",
+        "hat"
+      ],
+      [
+        "Ġf",
+        "ar"
+      ],
+      [
+        "Ġide",
+        "a"
+      ],
+      [
+        "Ġf",
+        "r"
+      ],
+      [
+        "o",
+        "on"
+      ],
+      [
+        "Ġno",
+        "dded"
+      ],
+      [
+        "Ġcol",
+        "or"
+      ],
+      [
+        "Ġwa",
+        "r"
+      ]
+    ]
+  }
+}

hf/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": [
+    "<|im_start|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<s>",
+  "tokenizer_class": "TokenizersBackend"
+}

requirements.txt

@@ -0,0 +1,2 @@
+torch
+transformers>=5.14.1